src/nepi/util/parsers/xml_parser.py

   1 #
   2 #    NEPI, a framework to manage network experiments
   3 #    Copyright (C) 2013 INRIA
   4 #
   5 #    This program is free software: you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License version 2 as
   7 #    published by the Free Software Foundation;
   8 #
   9 #    This program is distributed in the hope that it will be useful,
  10 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 #    GNU General Public License for more details.
  13 #
  14 #    You should have received a copy of the GNU General Public License
  15 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 #
  17 # Author: Alina Quereilhac <alina.quereilhac@inria.fr>
  18
  19 from __future__ import print_function
  20
  21 from nepi.util.netgraph import NetGraph, TopologyType
  22 from nepi.util.timefuncs import stformat, tsformat
  23
  24 from xml.dom import minidom
  25
  26 import datetime
  27 import sys
  28 import os
  29
  30 STRING = "string"
  31 BOOL = "bool"
  32 INTEGER = "integer"
  33 DOUBLE = "float"
  34
  35 from six import PY2
  36
  37 if PY2:
  38     # xxx old py2 code had a hack, that had 'latin1' hardcoded
  39     # as the encoding for 8-byte strings
  40     # this is very wrong; I keep it for now
  41     # but will probably remove it altogether some day
  42     def xmlencode(s):
  43         """xml encoder for python2"""
  44         if isinstance(s, str):
  45             rv = s.decode("latin1")
  46         if isinstance(s, datetime.datetime):
  47             rv = tsformat(s)
  48         elif not isinstance(s, unicode):
  49             rv = unicode(s)
  50         else:
  51             rv = s
  52         return rv.replace(u'\x00',u'&#0000;')
  53 else:
  54     # use sys.getdefaultencoding() to decode bytes into string
  55     def xmlencode(s):
  56         """xml encoder for python3"""
  57         if isinstance(s, datetime.datetime):
  58             rv = tsformat(s)
  59         elif isinstance(s, bytes):
  60             rv = s.decode(sys.getdefaultencoding())
  61         else:
  62             rv = s
  63         return rv.replace('\x00', '&#0000;')
  64
  65 def xmldecode(s, cast = str):
  66     if s is None:
  67         return None
  68     if PY2:
  69         ret = s.replace(u'&#0000', u'\x00').encode("ascii")
  70     else:
  71         ret = s.replace('&#0000', '\x00')
  72     ret = cast(ret)
  73     return ret
  74
  75 def from_type(value):
  76     if isinstance(value, bool):
  77         return BOOL
  78     if isinstance(value, int):
  79         return INTEGER
  80     if isinstance(value, float):
  81         return DOUBLE
  82
  83     return STRING
  84
  85 def to_type(type, value):
  86     if not value:
  87         return value
  88
  89     if type == STRING:
  90         return str(value)
  91     if type == BOOL:
  92         return value == "True"
  93     if type == INTEGER:
  94         return int(value)
  95     if type == DOUBLE:
  96         return float(value)
  97
  98 class ECXMLParser(object):
  99     def to_xml(self, ec):
 100
 101         doc = minidom.Document()
 102
 103         self._ec_to_xml(doc, ec)
 104
 105         try:
 106             xml = doc.toprettyxml(indent="    ", encoding="UTF-8")
 107         except:
 108             print("Oops: generating XML from %s" % (data,), file=sys.stderr)
 109             raise
 110
 111         return xml
 112
 113     def _ec_to_xml(self, doc, ec):
 114         ecnode = doc.createElement("experiment")
 115         ecnode.setAttribute("exp_id", xmlencode(ec.exp_id))
 116         ecnode.setAttribute("run_id", xmlencode(ec.run_id))
 117         ecnode.setAttribute("nthreads", xmlencode(ec.nthreads))
 118         ecnode.setAttribute("local_dir", xmlencode(ec.local_dir))
 119         doc.appendChild(ecnode)
 120
 121         if ec.netgraph != None:
 122             self._netgraph_to_xml(doc, ecnode, ec)
 123
 124         rmsnode = doc.createElement("rms")
 125         ecnode.appendChild(rmsnode)
 126
 127         for guid, rm in ec._resources.items():
 128             self._rm_to_xml(doc, rmsnode, ec, guid, rm)
 129
 130         return doc
 131
 132     def _netgraph_to_xml(self, doc, ecnode, ec):
 133         ngnode = doc.createElement("topology")
 134         ngnode.setAttribute("topo-type", xmlencode(ec.netgraph.topo_type))
 135         ecnode.appendChild(ngnode)
 136
 137         self. _netgraph_nodes_to_xml(doc, ngnode, ec)
 138         self. _netgraph_edges_to_xml(doc, ngnode, ec)
 139
 140     def _netgraph_nodes_to_xml(self, doc, ngnode, ec):
 141         ngnsnode = doc.createElement("nodes")
 142         ngnode.appendChild(ngnsnode)
 143
 144         for nid in ec.netgraph.nodes():
 145             ngnnode = doc.createElement("node")
 146             ngnnode.setAttribute("nid", xmlencode(nid))
 147             ngnnode.setAttribute("nid-type", from_type(nid))
 148             ngnsnode.appendChild(ngnnode)
 149
 150             # Mark ources and targets
 151             if ec.netgraph.is_source(nid):
 152                 ngnnode.setAttribute("source", xmlencode(True))
 153
 154             if ec.netgraph.is_target(nid):
 155                 ngnnode.setAttribute("target", xmlencode(True))
 156
 157             # Node annotations
 158             annosnode = doc.createElement("node-annotations")
 159             add_annotations = False
 160             for name in ec.netgraph.node_annotations(nid):
 161                 add_annotations = True
 162                 value = ec.netgraph.node_annotation(nid, name)
 163                 annonode = doc.createElement("node-annotation")
 164                 annonode.setAttribute("name", xmlencode(name))
 165                 annonode.setAttribute("value", xmlencode(value))
 166                 annonode.setAttribute("type", from_type(value))
 167                 annosnode.appendChild(annonode)
 168
 169             if add_annotations:
 170                 ngnnode.appendChild(annosnode)
 171
 172     def _netgraph_edges_to_xml(self, doc, ngnode, ec):
 173         ngesnode = doc.createElement("edges")
 174         ngnode.appendChild(ngesnode)
 175
 176         for nid1, nid2 in ec.netgraph.edges():
 177             ngenode = doc.createElement("edge")
 178             ngenode.setAttribute("nid1", xmlencode(nid1))
 179             ngenode.setAttribute("nid1-type", from_type(nid1))
 180             ngenode.setAttribute("nid2", xmlencode(nid2))
 181             ngenode.setAttribute("nid2-type", from_type(nid2))
 182             ngesnode.appendChild(ngenode)
 183
 184             # Edge annotations
 185             annosnode = doc.createElement("edge-annotations")
 186             add_annotations = False
 187             for name in ec.netgraph.edge_annotations(nid1, nid2):
 188                 add_annotations = True
 189                 value = ec.netgraph.edge_annotation(nid1, nid2, name)
 190                 annonode = doc.createElement("edge-annotation")
 191                 annonode.setAttribute("name", xmlencode(name))
 192                 annonode.setAttribute("value", xmlencode(value))
 193                 annonode.setAttribute("type", from_type(value))
 194                 annosnode.appendChild(annonode)
 195
 196             if add_annotations:
 197                 ngenode.appendChild(annosnode)
 198
 199     def _rm_to_xml(self, doc, rmsnode, ec, guid, rm):
 200         rmnode = doc.createElement("rm")
 201         rmnode.setAttribute("guid", xmlencode(guid))
 202         rmnode.setAttribute("rtype", xmlencode(rm._rtype))
 203         rmnode.setAttribute("state", xmlencode(rm._state))
 204         if rm._start_time:
 205             rmnode.setAttribute("start_time", xmlencode(rm._start_time))
 206         if rm._stop_time:
 207             rmnode.setAttribute("stop_time", xmlencode(rm._stop_time))
 208         if rm._discover_time:
 209             rmnode.setAttribute("discover_time", xmlencode(rm._discover_time))
 210         if rm._provision_time:
 211             rmnode.setAttribute("provision_time", xmlencode(rm._provision_time))
 212         if rm._ready_time:
 213             rmnode.setAttribute("ready_time", xmlencode(rm._ready_time))
 214         if rm._release_time:
 215             rmnode.setAttribute("release_time", xmlencode(rm._release_time))
 216         if rm._failed_time:
 217             rmnode.setAttribute("failed_time", xmlencode(rm._failed_time))
 218         rmsnode.appendChild(rmnode)
 219
 220         anode = doc.createElement("attributes")
 221         attributes = False
 222
 223         for attr in rm._attrs.values():
 224             if attr.has_changed:
 225                 attributes = True
 226                 aanode = doc.createElement("attribute")
 227                 aanode.setAttribute("name", xmlencode(attr.name))
 228                 aanode.setAttribute("value", xmlencode(attr.value))
 229                 aanode.setAttribute("type", from_type(attr.value))
 230                 anode.appendChild(aanode)
 231
 232         if attributes:
 233             rmnode.appendChild(anode)
 234
 235         cnode = doc.createElement("connections")
 236         connections = False
 237
 238         for guid in rm._connections:
 239             connections = True
 240             ccnode = doc.createElement("connection")
 241             ccnode.setAttribute("guid", xmlencode(guid))
 242             cnode.appendChild(ccnode)
 243
 244         if connections:
 245            rmnode.appendChild(cnode)
 246
 247         cnnode = doc.createElement("conditions")
 248         conditions = False
 249
 250         for action, conds in rm._conditions.items():
 251             conditions = True
 252             for (group, state, time) in conds:
 253                 ccnnode = doc.createElement("condition")
 254                 ccnnode.setAttribute("action", xmlencode(action))
 255                 ccnnode.setAttribute("group", xmlencode(group))
 256                 ccnnode.setAttribute("state", xmlencode(state))
 257                 ccnnode.setAttribute("time", xmlencode(time))
 258                 cnnode.appendChild(ccnnode)
 259
 260         if conditions:
 261            rmnode.appendChild(cnnode)
 262
 263         tnode = doc.createElement("traces")
 264         traces = False
 265
 266         for trace in rm._trcs.values():
 267             if trace.enabled:
 268                 traces = True
 269                 ttnode = doc.createElement("trace")
 270                 ttnode.setAttribute("name", xmlencode(trace.name))
 271                 tnode.appendChild(ttnode)
 272
 273         if traces:
 274             rmnode.appendChild(tnode)
 275
 276     def from_xml(self, xml):
 277         doc = minidom.parseString(xml)
 278         return self._ec_from_xml(doc)
 279
 280     def _ec_from_xml(self, doc):
 281         from nepi.execution.ec import ExperimentController
 282         ec = None
 283
 284         ecnode_list = doc.getElementsByTagName("experiment")
 285         for ecnode in ecnode_list:
 286             if ecnode.nodeType == doc.ELEMENT_NODE:
 287                 exp_id = xmldecode(ecnode.getAttribute("exp_id"))
 288                 run_id = xmldecode(ecnode.getAttribute("run_id"))
 289                 local_dir = xmldecode(ecnode.getAttribute("local_dir"))
 290
 291                 # Configure number of preocessing threads
 292                 nthreads = xmldecode(ecnode.getAttribute("nthreads"))
 293                 os.environ["NEPI_NTHREADS"] = nthreads
 294
 295                 # Deserialize netgraph
 296                 topology = None
 297                 topo_type = None
 298
 299                 netgraph = self._netgraph_from_xml(doc, ecnode)
 300
 301                 if netgraph:
 302                     topo_type = netgraph.topo_type
 303                     topology = netgraph.topology
 304
 305                 # Instantiate EC
 306                 ec = ExperimentController(exp_id = exp_id, local_dir = local_dir,
 307                         topology = topology, topo_type = topo_type)
 308
 309                 connections = set()
 310
 311                 rmsnode_list = ecnode.getElementsByTagName("rms")
 312                 if rmsnode_list:
 313                     rmnode_list = rmsnode_list[0].getElementsByTagName("rm")
 314                     for rmnode in rmnode_list:
 315                         if rmnode.nodeType == doc.ELEMENT_NODE:
 316                             self._rm_from_xml(doc, rmnode, ec, connections)
 317
 318                 for (guid1, guid2) in connections:
 319                     ec.register_connection(guid1, guid2)
 320
 321                 break
 322
 323         return ec
 324
 325     def _netgraph_from_xml(self, doc, ecnode):
 326         netgraph = None
 327
 328         topology = ecnode.getElementsByTagName("topology")
 329         if topology:
 330             topology = topology[0]
 331             topo_type = xmldecode(topology.getAttribute("topo-type"))
 332
 333             netgraph = NetGraph(topo_type = topo_type)
 334
 335             ngnsnode_list = topology.getElementsByTagName("nodes")
 336             if ngnsnode_list:
 337                 ngnsnode = ngnsnode_list[0].getElementsByTagName("node")
 338                 for ngnnode in ngnsnode:
 339                     nid = xmldecode(ngnnode.getAttribute("nid"))
 340                     tipe = xmldecode(ngnnode.getAttribute("nid-type"))
 341                     nid = to_type(tipe, nid)
 342                     netgraph.add_node(nid)
 343
 344                     if ngnnode.hasAttribute("source"):
 345                         netgraph.set_source(nid)
 346                     if ngnnode.hasAttribute("target"):
 347                         netgraph.set_target(nid)
 348
 349                     annosnode_list = ngnnode.getElementsByTagName("node-annotations")
 350
 351                     if annosnode_list:
 352                         annosnode = annosnode_list[0].getElementsByTagName("node-annotation")
 353                         for annonode in annosnode:
 354                             name = xmldecode(annonode.getAttribute("name"))
 355
 356                             if name == "ips":
 357                                 ips = xmldecode(annonode.getAttribute("value"), eval) # list
 358                                 for ip in ips:
 359                                     netgraph.annotate_node_ip(nid, ip)
 360                             else:
 361                                 value = xmldecode(annonode.getAttribute("value"))
 362                                 tipe = xmldecode(annonode.getAttribute("type"))
 363                                 value = to_type(tipe, value)
 364                                 netgraph.annotate_node(nid, name, value)
 365
 366             ngesnode_list = topology.getElementsByTagName("edges")
 367             if ngesnode_list:
 368                 ngesnode = ngesnode_list[0].getElementsByTagName("edge")
 369                 for ngenode in ngesnode:
 370                     nid1 = xmldecode(ngenode.getAttribute("nid1"))
 371                     tipe1 = xmldecode(ngenode.getAttribute("nid1-type"))
 372                     nid1 = to_type(tipe1, nid1)
 373
 374                     nid2 = xmldecode(ngenode.getAttribute("nid2"))
 375                     tipe2 = xmldecode(ngenode.getAttribute("nid2-type"))
 376                     nid2 = to_type(tipe2, nid2)
 377
 378                     netgraph.add_edge(nid1, nid2)
 379
 380                     annosnode_list = ngenode.getElementsByTagName("edge-annotations")
 381                     if annosnode_list:
 382                         annosnode = annosnode_list[0].getElementsByTagName("edge-annotation")
 383                         for annonode in annosnode:
 384                             name = xmldecode(annonode.getAttribute("name"))
 385
 386                             if name == "net":
 387                                 net = xmldecode(annonode.getAttribute("value"), eval) # dict
 388                                 netgraph.annotate_edge_net(nid1, nid2, net[nid1], net[nid2],
 389                                         net["mask"], net["network"], net["prefix"])
 390                             else:
 391                                 value = xmldecode(annonode.getAttribute("value"))
 392                                 tipe = xmldecode(annonode.getAttribute("type"))
 393                                 value = to_type(tipe, value)
 394                                 netgraph.annotate_edge(nid1, nid2, name, value)
 395         return netgraph
 396
 397     def _rm_from_xml(self, doc, rmnode, ec, connections):
 398         start_time = None
 399         stop_time = None
 400         discover_time = None
 401         provision_time = None
 402         ready_time = None
 403         release_time = None
 404         failed_time = None
 405
 406         guid = xmldecode(rmnode.getAttribute("guid"), int)
 407         rtype = xmldecode(rmnode.getAttribute("rtype"))
 408
 409         # FOR NOW ONLY STATE NEW IS ALLOWED
 410         state = 0
 411         """
 412         state = xmldecode(rmnode.getAttribute("state"), int)
 413
 414         if rmnode.hasAttribute("start_time"):
 415             start_time = xmldecode(rmnode.getAttribute("start_time"),
 416                     datetime.datetime)
 417         if rmnode.hasAttribute("stop_time"):
 418             stop_time = xmldecode(rmnode.getAttribute("stop_time"),
 419                     datetime.datetime)
 420         if rmnode.hasAttribute("discover_time"):
 421             dicover_time = xmldecode(rmnode.getAttribute("discover_time"),
 422                     datetime.datetime)
 423         if rmnode.hasAttribute("provision_time"):
 424             provision_time = xmldecode(rmnode.getAttribute("provision_time"),
 425                     datetime.datetime)
 426         if rmnode.hasAttribute("ready_time"):
 427             ready_time = xmldecode(rmnode.getAttribute("ready_time"),
 428                     datetime.datetime)
 429         if rmnode.hasAttribute("release_time"):
 430             release_time = xmldecode(rmnode.getAttribute("release_time"),
 431                     datetime.datetime)
 432         if rmnode.hasAttribute("failed_time"):
 433             failed_time = xmldecode(rmnode.getAttribute("failed_time"),
 434                     datetime.datetime)
 435         """
 436
 437         ec.register_resource(rtype, guid = guid)
 438         rm = ec.get_resource(guid)
 439         rm.set_state_time(state, "_start_time", start_time)
 440         rm.set_state_time(state, "_stop_time", stop_time)
 441         rm.set_state_time(state, "_discover_time", discover_time)
 442         rm.set_state_time(state, "_provision_time", provision_time)
 443         rm.set_state_time(state, "_ready_time", ready_time)
 444         rm.set_state_time(state, "_release_time", release_time)
 445         rm.set_state_time(state, "_failed_time", failed_time)
 446
 447         anode_list = rmnode.getElementsByTagName("attributes")
 448         if anode_list:
 449             aanode_list = anode_list[0].getElementsByTagName("attribute")
 450             for aanode in aanode_list:
 451                 name = xmldecode(aanode.getAttribute("name"))
 452                 value = xmldecode(aanode.getAttribute("value"))
 453                 tipe = xmldecode(aanode.getAttribute("type"))
 454                 value = to_type(tipe, value)
 455                 rm.set(name, value)
 456
 457         cnode_list = rmnode.getElementsByTagName("connections")
 458         if cnode_list:
 459             ccnode_list = cnode_list[0].getElementsByTagName("connection")
 460             for ccnode in ccnode_list:
 461                 guid2 = xmldecode(ccnode.getAttribute("guid"), int)
 462                 connections.add((guid, guid2))
 463
 464         tnode_list = rmnode.getElementsByTagName("traces")
 465         if tnode_list:
 466             ttnode_list = tnode_list[0].getElementsByTagName("trace")
 467             for ttnode in ttnode_list:
 468                 name = xmldecode(ttnode.getAttribute("name"))
 469                 ec.enable_trace(guid, name)
 470
 471         cnnode_list = rmnode.getElementsByTagName("conditions")
 472         if cnnode_list:
 473             ccnnode_list = cnnode_list[0].getElementsByTagName("condition")
 474             for ccnnode in ccnnode_list:
 475                 action = xmldecode(ccnnode.getAttribute("action"), int)
 476                 group = xmldecode(ccnnode.getAttribute("group"), eval) # list
 477                 state = xmldecode(ccnnode.getAttribute("state"), int)
 478                 time = xmldecode(ccnnode.getAttribute("time"))
 479                 time = to_type('STRING', time)
 480                 ec.register_condition(guid, action, group, state, time = time)
 481