src/nepi/util/parsers/xml_parser.py

   1 #
   2 #    NEPI, a framework to manage network experiments
   3 #    Copyright (C) 2013 INRIA
   4 #
   5 #    This program is free software: you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation, either version 3 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 #
  18 # Author: Alina Quereilhac <alina.quereilhac@inria.fr>
  19
  20 from nepi.util.netgraph import NetGraph, TopologyType
  21 from nepi.util.timefuncs import stformat, tsformat
  22
  23 from xml.dom import minidom
  24
  25 import datetime
  26 import sys
  27 import os
  28
  29 STRING = "string"
  30 BOOL = "bool"
  31 INTEGER = "integer"
  32 DOUBLE = "float"
  33
  34 def xmlencode(s):
  35     if isinstance(s, str):
  36         rv = s.decode("latin1")
  37     if isinstance(s, datetime.datetime):
  38         rv = tsformat(s)
  39     elif not isinstance(s, unicode):
  40         rv = unicode(s)
  41     else:
  42         rv = s
  43     return rv.replace(u'\x00',u'&#0000;')
  44
  45 def xmldecode(s, cast = str):
  46     ret = s.replace(u'&#0000',u'\x00').encode("ascii")
  47     ret = cast(ret)
  48     if s == "None":
  49         return None
  50     return ret
  51
  52 def from_type(value):
  53     if isinstance(value, bool):
  54         return BOOL
  55     if isinstance(value, int):
  56         return INTEGER
  57     if isinstance(value, float):
  58         return DOUBLE
  59
  60     return STRING
  61
  62 def to_type(type, value):
  63     if not value:
  64         return value
  65
  66     if type == STRING:
  67         return str(value)
  68     if type == BOOL:
  69         return value == "True"
  70     if type == INTEGER:
  71         return int(value)
  72     if type == DOUBLE:
  73         return float(value)
  74
  75 class ECXMLParser(object):
  76     def to_xml(self, ec):
  77
  78         doc = minidom.Document()
  79
  80         self._ec_to_xml(doc, ec)
  81
  82         try:
  83             xml = doc.toprettyxml(indent="    ", encoding="UTF-8")
  84         except:
  85             print >>sys.stderr, "Oops: generating XML from %s" % (data,)
  86             raise
  87
  88         return xml
  89
  90     def _ec_to_xml(self, doc, ec):
  91         ecnode = doc.createElement("experiment")
  92         ecnode.setAttribute("exp_id", xmlencode(ec.exp_id))
  93         ecnode.setAttribute("run_id", xmlencode(ec.run_id))
  94         ecnode.setAttribute("nthreads", xmlencode(ec.nthreads))
  95         ecnode.setAttribute("local_dir", xmlencode(ec.local_dir))
  96         doc.appendChild(ecnode)
  97
  98         if ec.netgraph != None:
  99             self._netgraph_to_xml(doc, ecnode, ec)
 100
 101         rmsnode = doc.createElement("rms")
 102         ecnode.appendChild(rmsnode)
 103
 104         for guid, rm in ec._resources.iteritems():
 105             self._rm_to_xml(doc, rmsnode, ec, guid, rm)
 106
 107         return doc
 108
 109     def _netgraph_to_xml(self, doc, ecnode, ec):
 110         ngnode = doc.createElement("topology")
 111         ngnode.setAttribute("topo-type", xmlencode(ec.netgraph.topo_type))
 112         ecnode.appendChild(ngnode)
 113
 114         self. _netgraph_nodes_to_xml(doc, ngnode, ec)
 115         self. _netgraph_edges_to_xml(doc, ngnode, ec)
 116
 117     def _netgraph_nodes_to_xml(self, doc, ngnode, ec):
 118         ngnsnode = doc.createElement("nodes")
 119         ngnode.appendChild(ngnsnode)
 120
 121         for nid in ec.netgraph.nodes():
 122             ngnnode = doc.createElement("node")
 123             ngnnode.setAttribute("nid", xmlencode(nid))
 124             ngnnode.setAttribute("nid-type", from_type(nid))
 125             ngnsnode.appendChild(ngnnode)
 126
 127             # Mark ources and targets
 128             if ec.netgraph.is_source(nid):
 129                 ngnnode.setAttribute("source", xmlencode(True))
 130
 131             if ec.netgraph.is_target(nid):
 132                 ngnnode.setAttribute("target", xmlencode(True))
 133
 134             # Node annotations
 135             annosnode = doc.createElement("node-annotations")
 136             add_annotations = False
 137             for name in ec.netgraph.node_annotations(nid):
 138                 add_annotations = True
 139                 value = ec.netgraph.node_annotation(nid, name)
 140                 annonode = doc.createElement("node-annotation")
 141                 annonode.setAttribute("name", xmlencode(name))
 142                 annonode.setAttribute("value", xmlencode(value))
 143                 annonode.setAttribute("type", from_type(value))
 144                 annosnode.appendChild(annonode)
 145
 146             if add_annotations:
 147                 ngnnode.appendChild(annosnode)
 148
 149     def _netgraph_edges_to_xml(self, doc, ngnode, ec):
 150         ngesnode = doc.createElement("edges")
 151         ngnode.appendChild(ngesnode)
 152
 153         for nid1, nid2 in ec.netgraph.edges():
 154             ngenode = doc.createElement("edge")
 155             ngenode.setAttribute("nid1", xmlencode(nid1))
 156             ngenode.setAttribute("nid1-type", from_type(nid1))
 157             ngenode.setAttribute("nid2", xmlencode(nid2))
 158             ngenode.setAttribute("nid2-type", from_type(nid2))
 159             ngesnode.appendChild(ngenode)
 160
 161             # Edge annotations
 162             annosnode = doc.createElement("edge-annotations")
 163             add_annotations = False
 164             for name in ec.netgraph.edge_annotations(nid1, nid2):
 165                 add_annotations = True
 166                 value = ec.netgraph.edge_annotation(nid1, nid2, name)
 167                 annonode = doc.createElement("edge-annotation")
 168                 annonode.setAttribute("name", xmlencode(name))
 169                 annonode.setAttribute("value", xmlencode(value))
 170                 annonode.setAttribute("type", from_type(value))
 171                 annosnode.appendChild(annonode)
 172
 173             if add_annotations:
 174                 ngenode.appendChild(annosnode)
 175
 176     def _rm_to_xml(self, doc, rmsnode, ec, guid, rm):
 177         rmnode = doc.createElement("rm")
 178         rmnode.setAttribute("guid", xmlencode(guid))
 179         rmnode.setAttribute("rtype", xmlencode(rm._rtype))
 180         rmnode.setAttribute("state", xmlencode(rm._state))
 181         if rm._start_time:
 182             rmnode.setAttribute("start_time", xmlencode(rm._start_time))
 183         if rm._stop_time:
 184             rmnode.setAttribute("stop_time", xmlencode(rm._stop_time))
 185         if rm._discover_time:
 186             rmnode.setAttribute("discover_time", xmlencode(rm._discover_time))
 187         if rm._provision_time:
 188             rmnode.setAttribute("provision_time", xmlencode(rm._provision_time))
 189         if rm._ready_time:
 190             rmnode.setAttribute("ready_time", xmlencode(rm._ready_time))
 191         if rm._release_time:
 192             rmnode.setAttribute("release_time", xmlencode(rm._release_time))
 193         if rm._failed_time:
 194             rmnode.setAttribute("failed_time", xmlencode(rm._failed_time))
 195         rmsnode.appendChild(rmnode)
 196
 197         anode = doc.createElement("attributes")
 198         attributes = False
 199
 200         for attr in rm._attrs.values():
 201             if attr.has_changed:
 202                 attributes = True
 203                 aanode = doc.createElement("attribute")
 204                 aanode.setAttribute("name", xmlencode(attr.name))
 205                 aanode.setAttribute("value", xmlencode(attr.value))
 206                 aanode.setAttribute("type", from_type(attr.value))
 207                 anode.appendChild(aanode)
 208
 209         if attributes:
 210             rmnode.appendChild(anode)
 211
 212         cnode = doc.createElement("connections")
 213         connections = False
 214
 215         for guid in rm._connections:
 216             connections = True
 217             ccnode = doc.createElement("connection")
 218             ccnode.setAttribute("guid", xmlencode(guid))
 219             cnode.appendChild(ccnode)
 220
 221         if connections:
 222            rmnode.appendChild(cnode)
 223
 224         cnnode = doc.createElement("conditions")
 225         conditions = False
 226
 227         for action, conds in rm._conditions.iteritems():
 228             conditions = True
 229             for (group, state, time) in conds:
 230                 ccnnode = doc.createElement("condition")
 231                 ccnnode.setAttribute("action", xmlencode(action))
 232                 ccnnode.setAttribute("group", xmlencode(group))
 233                 ccnnode.setAttribute("state", xmlencode(state))
 234                 ccnnode.setAttribute("time", xmlencode(time))
 235                 cnnode.appendChild(ccnnode)
 236
 237         if conditions:
 238            rmnode.appendChild(cnnode)
 239
 240         tnode = doc.createElement("traces")
 241         traces = False
 242
 243         for trace in rm._trcs.values():
 244             if trace.enabled:
 245                 traces = True
 246                 ttnode = doc.createElement("trace")
 247                 ttnode.setAttribute("name", xmlencode(trace.name))
 248                 tnode.appendChild(ttnode)
 249
 250         if traces:
 251             rmnode.appendChild(tnode)
 252
 253     def from_xml(self, xml):
 254         doc = minidom.parseString(xml)
 255         return self._ec_from_xml(doc)
 256
 257     def _ec_from_xml(self, doc):
 258         from nepi.execution.ec import ExperimentController
 259         ec = None
 260
 261         ecnode_list = doc.getElementsByTagName("experiment")
 262         for ecnode in ecnode_list:
 263             if ecnode.nodeType == doc.ELEMENT_NODE:
 264                 exp_id = xmldecode(ecnode.getAttribute("exp_id"))
 265                 run_id = xmldecode(ecnode.getAttribute("run_id"))
 266                 local_dir = xmldecode(ecnode.getAttribute("local_dir"))
 267
 268                 # Configure number of preocessing threads
 269                 nthreads = xmldecode(ecnode.getAttribute("nthreads"))
 270                 os.environ["NEPI_NTHREADS"] = nthreads
 271
 272                 # Deserialize netgraph
 273                 topology = None
 274                 topo_type = None
 275
 276                 netgraph = self._netgraph_from_xml(doc, ecnode)
 277
 278                 if netgraph:
 279                     topo_type = netgraph.topo_type
 280                     topology = netgraph.topology
 281
 282                 # Instantiate EC
 283                 ec = ExperimentController(exp_id = exp_id, local_dir = local_dir,
 284                         topology = topology, topo_type = topo_type)
 285
 286                 connections = set()
 287
 288                 rmsnode_list = ecnode.getElementsByTagName("rms")
 289                 if rmsnode_list:
 290                     rmnode_list = rmsnode_list[0].getElementsByTagName("rm")
 291                     for rmnode in rmnode_list:
 292                         if rmnode.nodeType == doc.ELEMENT_NODE:
 293                             self._rm_from_xml(doc, rmnode, ec, connections)
 294
 295                 for (guid1, guid2) in connections:
 296                     ec.register_connection(guid1, guid2)
 297
 298                 break
 299
 300         return ec
 301
 302     def _netgraph_from_xml(self, doc, ecnode):
 303         netgraph = None
 304
 305         topology = ecnode.getElementsByTagName("topology")
 306         if topology:
 307             topology = topology[0]
 308             topo_type = xmldecode(topology.getAttribute("topo-type"))
 309
 310             netgraph = NetGraph(topo_type = topo_type)
 311
 312             ngnsnode_list = topology.getElementsByTagName("nodes")
 313             if ngnsnode_list:
 314                 ngnsnode = ngnsnode_list[0].getElementsByTagName("node")
 315                 for ngnnode in ngnsnode:
 316                     nid = xmldecode(ngnnode.getAttribute("nid"))
 317                     tipe = xmldecode(ngnnode.getAttribute("nid-type"))
 318                     nid = to_type(tipe, nid)
 319                     netgraph.add_node(nid)
 320
 321                     if ngnnode.hasAttribute("source"):
 322                         netgraph.set_source(nid)
 323                     if ngnnode.hasAttribute("target"):
 324                         netgraph.set_target(nid)
 325
 326                     annosnode_list = ngnnode.getElementsByTagName("node-annotations")
 327
 328                     if annosnode_list:
 329                         annosnode = annosnode_list[0].getElementsByTagName("node-annotation")
 330                         for annonode in annosnode:
 331                             name = xmldecode(annonode.getAttribute("name"))
 332
 333                             if name == "ips":
 334                                 ips = xmldecode(annonode.getAttribute("value"), eval) # list
 335                                 for ip in ips:
 336                                     netgraph.annotate_node_ip(nid, ip)
 337                             else:
 338                                 value = xmldecode(annonode.getAttribute("value"))
 339                                 tipe = xmldecode(annonode.getAttribute("type"))
 340                                 value = to_type(tipe, value)
 341                                 netgraph.annotate_node(nid, name, value)
 342
 343             ngesnode_list = topology.getElementsByTagName("edges")
 344             if ngesnode_list:
 345                 ngesnode = ngesnode_list[0].getElementsByTagName("edge")
 346                 for ngenode in ngesnode:
 347                     nid1 = xmldecode(ngenode.getAttribute("nid1"))
 348                     tipe1 = xmldecode(ngenode.getAttribute("nid1-type"))
 349                     nid1 = to_type(tipe1, nid1)
 350
 351                     nid2 = xmldecode(ngenode.getAttribute("nid2"))
 352                     tipe2 = xmldecode(ngenode.getAttribute("nid2-type"))
 353                     nid2 = to_type(tipe2, nid2)
 354
 355                     netgraph.add_edge(nid1, nid2)
 356
 357                     annosnode_list = ngenode.getElementsByTagName("edge-annotations")
 358                     if annosnode_list:
 359                         annosnode = annosnode_list[0].getElementsByTagName("edge-annotation")
 360                         for annonode in annosnode:
 361                             name = xmldecode(annonode.getAttribute("name"))
 362
 363                             if name == "net":
 364                                 net = xmldecode(annonode.getAttribute("value"), eval) # dict
 365                                 netgraph.annotate_edge_net(nid1, nid2, net[nid1], net[nid2],
 366                                         net["mask"], net["network"], net["prefix"])
 367                             else:
 368                                 value = xmldecode(annonode.getAttribute("value"))
 369                                 tipe = xmldecode(annonode.getAttribute("type"))
 370                                 value = to_type(tipe, value)
 371                                 netgraph.annotate_edge(nid1, nid2, name, value)
 372         return netgraph
 373
 374     def _rm_from_xml(self, doc, rmnode, ec, connections):
 375         start_time = None
 376         stop_time = None
 377         discover_time = None
 378         provision_time = None
 379         ready_time = None
 380         release_time = None
 381         failed_time = None
 382
 383         guid = xmldecode(rmnode.getAttribute("guid"), int)
 384         rtype = xmldecode(rmnode.getAttribute("rtype"))
 385
 386         # FOR NOW ONLY STATE NEW IS ALLOWED
 387         state = 0
 388         """
 389         state = xmldecode(rmnode.getAttribute("state"), int)
 390
 391         if rmnode.hasAttribute("start_time"):
 392             start_time = xmldecode(rmnode.getAttribute("start_time"),
 393                     datetime.datetime)
 394         if rmnode.hasAttribute("stop_time"):
 395             stop_time = xmldecode(rmnode.getAttribute("stop_time"),
 396                     datetime.datetime)
 397         if rmnode.hasAttribute("discover_time"):
 398             dicover_time = xmldecode(rmnode.getAttribute("discover_time"),
 399                     datetime.datetime)
 400         if rmnode.hasAttribute("provision_time"):
 401             provision_time = xmldecode(rmnode.getAttribute("provision_time"),
 402                     datetime.datetime)
 403         if rmnode.hasAttribute("ready_time"):
 404             ready_time = xmldecode(rmnode.getAttribute("ready_time"),
 405                     datetime.datetime)
 406         if rmnode.hasAttribute("release_time"):
 407             release_time = xmldecode(rmnode.getAttribute("release_time"),
 408                     datetime.datetime)
 409         if rmnode.hasAttribute("failed_time"):
 410             failed_time = xmldecode(rmnode.getAttribute("failed_time"),
 411                     datetime.datetime)
 412         """
 413
 414         ec.register_resource(rtype, guid = guid)
 415         rm = ec.get_resource(guid)
 416         rm.set_state_time(state, "_start_time", start_time)
 417         rm.set_state_time(state, "_stop_time", stop_time)
 418         rm.set_state_time(state, "_discover_time", discover_time)
 419         rm.set_state_time(state, "_provision_time", provision_time)
 420         rm.set_state_time(state, "_ready_time", ready_time)
 421         rm.set_state_time(state, "_release_time", release_time)
 422         rm.set_state_time(state, "_failed_time", failed_time)
 423
 424         anode_list = rmnode.getElementsByTagName("attributes")
 425         if anode_list:
 426             aanode_list = anode_list[0].getElementsByTagName("attribute")
 427             for aanode in aanode_list:
 428                 name = xmldecode(aanode.getAttribute("name"))
 429                 value = xmldecode(aanode.getAttribute("value"))
 430                 tipe = xmldecode(aanode.getAttribute("type"))
 431                 value = to_type(tipe, value)
 432                 rm.set(name, value)
 433
 434         cnode_list = rmnode.getElementsByTagName("connections")
 435         if cnode_list:
 436             ccnode_list = cnode_list[0].getElementsByTagName("connection")
 437             for ccnode in ccnode_list:
 438                 guid2 = xmldecode(ccnode.getAttribute("guid"), int)
 439                 connections.add((guid, guid2))
 440
 441         tnode_list = rmnode.getElementsByTagName("traces")
 442         if tnode_list:
 443             ttnode_list = tnode_list[0].getElementsByTagName("trace")
 444             for ttnode in ttnode_list:
 445                 name = xmldecode(ttnode.getAttribute("name"))
 446                 ec.enable_trace(guid, name)
 447
 448         cnnode_list = rmnode.getElementsByTagName("conditions")
 449         if cnnode_list:
 450             ccnnode_list = cnnode_list[0].getElementsByTagName("condition")
 451             for ccnnode in ccnnode_list:
 452                 action = xmldecode(ccnnode.getAttribute("action"), int)
 453                 group = xmldecode(ccnnode.getAttribute("group"), eval) # list
 454                 state = xmldecode(ccnnode.getAttribute("state"), int)
 455                 time = xmldecode(ccnnode.getAttribute("time"))
 456                 time = to_type('STRING', time)
 457                 ec.register_condition(guid, action, group, state, time = time)
 458