sfa/util/rspecHelper.py

   1 #! /usr/bin/env python
   2
   3 import sys
   4
   5 from copy import deepcopy
   6 from lxml import etree
   7 from StringIO import StringIO
   8 from optparse import OptionParser
   9
  10 from sfa.util.faults import InvalidRSpec
  11 from sfa.util.sfalogging import logger
  12
  13 def merge_rspecs(rspecs):
  14     """
  15     Merge merge a list of RSpecs into 1 RSpec, and return the result.
  16     rspecs must be a valid RSpec string or list of RSpec strings.
  17     """
  18     if not rspecs or not isinstance(rspecs, list):
  19         return rspecs
  20
  21     # ugly hack to avoid sending the same info twice, when the call graph has dags
  22     known_networks={}
  23     def register_network (network):
  24         try:
  25             known_networks[network.get('name')]=True
  26         except:
  27             logger.error("merge_rspecs: cannot register network with no name in rspec")
  28             pass
  29     def is_registered_network (network):
  30         try:
  31             return network.get('name') in known_networks
  32         except:
  33             logger.error("merge_rspecs: cannot retrieve network with no name in rspec")
  34             return False
  35
  36     # the resulting tree
  37     rspec = None
  38     for input_rspec in rspecs:
  39         # ignore empty strings as returned with used call_ids
  40         if not input_rspec: continue
  41         try:
  42             tree = etree.parse(StringIO(input_rspec))
  43         except etree.XMLSyntaxError:
  44             # consider failing silently here
  45             logger.log_exc("merge_rspecs, parse error")
  46             message = str(sys.exc_info()[1]) + ' with ' + input_rspec
  47             raise InvalidRSpec(message)
  48
  49         root = tree.getroot()
  50         if not root.get("type") in ["SFA"]:
  51             logger.error("merge_rspecs: unexpected type for rspec root, %s"%root.get('type'))
  52             continue
  53         if rspec == None:
  54             # we scan the first input, register all networks
  55             # in addition we remove duplicates - needed until everyone runs 1.0-10
  56             rspec = root
  57             for network in root.iterfind("./network"):
  58                 if not is_registered_network(network):
  59                     register_network(network)
  60                 else:
  61                     # duplicate in the first input - trash it
  62                     root.remove(network)
  63         else:
  64             for network in root.iterfind("./network"):
  65                 if not is_registered_network(network):
  66                     rspec.append(deepcopy(network))
  67                     register_network(network)
  68             for request in root.iterfind("./request"):
  69                 rspec.append(deepcopy(request))
  70     return etree.tostring(rspec, xml_declaration=True, pretty_print=True)
  71
  72 class RSpec:
  73     def __init__(self, xml):
  74         parser = etree.XMLParser(remove_blank_text=True)
  75         tree = etree.parse(StringIO(xml), parser)
  76         self.rspec = tree.getroot()
  77
  78         # If there is only one network in the rspec, make it the default
  79         self.network = None
  80         networks = self.get_network_list()
  81         if len(networks) == 1:
  82             self.network = networks[0]
  83
  84     # Thierry : need this to locate hostname even if several networks
  85     def get_node_element(self, hostname, network=None):
  86         if network == None and self.network:
  87             network = self.network
  88         if network != None:
  89             names = self.rspec.iterfind("./network[@name='%s']/site/node/hostname" % network)
  90         else:
  91             names = self.rspec.iterfind("./network/site/node/hostname")
  92         for name in names:
  93             if name.text == hostname:
  94                 return name.getparent()
  95         return None
  96
  97     # Thierry : need this to return all nodes in all networks
  98     def get_node_list(self, network=None):
  99         if network == None and self.network:
 100             network = self.network
 101         if network != None:
 102             return self.rspec.xpath("./network[@name='%s']/site/node/hostname/text()" % network)
 103         else:
 104             return self.rspec.xpath("./network/site/node/hostname/text()")
 105
 106     def get_network_list(self):
 107         return self.rspec.xpath("./network[@name]/@name")
 108
 109     def get_sliver_list(self, network=None):
 110         if network == None:
 111             network = self.network
 112         result = self.rspec.xpath("./network[@name='%s']/site/node[sliver]/hostname/text()" % network)
 113         return result
 114
 115     def get_available_node_list(self, network=None):
 116         if network == None:
 117             network = self.network
 118         result = self.rspec.xpath("./network[@name='%s']/site/node[not(sliver)]/hostname/text()" % network)
 119         return result
 120
 121     def add_sliver(self, hostname, network=None):
 122         if network == None:
 123             network = self.network
 124         node = self.get_node_element(hostname, network)
 125         etree.SubElement(node, "sliver")
 126
 127     def remove_sliver(self, hostname, network=None):
 128         if network == None:
 129             network = self.network
 130         node = self.get_node_element(hostname, network)
 131         node.remove(node.find("sliver"))
 132
 133     def attributes_list(self, elem):
 134         opts = []
 135         if elem is not None:
 136             for e in elem:
 137                 opts.append((e.tag, e.text))
 138         return opts
 139
 140     def get_default_sliver_attributes(self, network=None):
 141         if network == None:
 142             network = self.network
 143         defaults = self.rspec.find("./network[@name='%s']/sliver_defaults" % network)
 144         return self.attributes_list(defaults)
 145
 146     def get_sliver_attributes(self, hostname, network=None):
 147         if network == None:
 148             network = self.network
 149         node = self.get_node_element(hostname, network)
 150         sliver = node.find("sliver")
 151         return self.attributes_list(sliver)
 152
 153     def add_attribute(self, elem, name, value):
 154         opt = etree.SubElement(elem, name)
 155         opt.text = value
 156
 157     def add_default_sliver_attribute(self, name, value, network=None):
 158         if network == None:
 159             network = self.network
 160         defaults = self.rspec.find("./network[@name='%s']/sliver_defaults" % network)
 161         if defaults is None:
 162             defaults = etree.Element("sliver_defaults")
 163             network = self.rspec.find("./network[@name='%s']" % network)
 164             network.insert(0, defaults)
 165         self.add_attribute(defaults, name, value)
 166
 167     def add_sliver_attribute(self, hostname, name, value, network=None):
 168         if network == None:
 169             network = self.network
 170         node = self.get_node_element(hostname, network)
 171         sliver = node.find("sliver")
 172         self.add_attribute(sliver, name, value)
 173
 174     def remove_attribute(self, elem, name, value):
 175         if elem is not None:
 176             opts = elem.iterfind(name)
 177             if opts is not None:
 178                 for opt in opts:
 179                     if opt.text == value:
 180                         elem.remove(opt)
 181
 182     def remove_default_sliver_attribute(self, name, value, network=None):
 183         if network == None:
 184             network = self.network
 185         defaults = self.rspec.find("./network[@name='%s']/sliver_defaults" % network)
 186         self.remove_attribute(defaults, name, value)
 187
 188     def remove_sliver_attribute(self, hostname, name, value, network=None):
 189         if network == None:
 190             network = self.network
 191         node = self.get_node_element(hostname, network)
 192         sliver = node.find("sliver")
 193         self.remove_attribute(sliver, name, value)
 194
 195     def get_site_nodes(self, siteid, network=None):
 196         if network == None:
 197             network = self.network
 198         query = './network[@name="%s"]/site[@id="%s"]/node/hostname/text()' % (network, siteid)
 199         result = self.rspec.xpath(query)
 200         return result
 201
 202     def get_link_list(self, network=None):
 203         if network == None:
 204             network = self.network
 205         linklist = []
 206         links = self.rspec.iterfind("./network[@name='%s']/link" % network)
 207         for link in links:
 208             (end1, end2) = link.get("endpoints").split()
 209             name = link.find("description")
 210             linklist.append((name.text,
 211                              self.get_site_nodes(end1, network),
 212                              self.get_site_nodes(end2, network)))
 213         return linklist
 214
 215     def get_vlink_list(self, network=None):
 216         if network == None:
 217             network = self.network
 218         vlinklist = []
 219         vlinks = self.rspec.iterfind("./network[@name='%s']//vlink" % network)
 220         for vlink in vlinks:
 221             endpoints = vlink.get("endpoints")
 222             (end1, end2) = endpoints.split()
 223             query = './network[@name="%s"]//node[@id="%s"]/hostname/text()' % network
 224             node1 = self.rspec.xpath(query % end1)[0]
 225             node2 = self.rspec.xpath(query % end2)[0]
 226             desc = "%s <--> %s" % (node1, node2)
 227             kbps = vlink.find("kbps")
 228             vlinklist.append((endpoints, desc, kbps.text))
 229         return vlinklist
 230
 231     def query_links(self, fromnode, tonode, network=None):
 232         if network == None:
 233             network = self.network
 234         fromsite = fromnode.getparent()
 235         tosite = tonode.getparent()
 236         fromid = fromsite.get("id")
 237         toid = tosite.get("id")
 238
 239         query = "./network[@name='%s']/link[@endpoints = '%s %s']" % (network, fromid, toid)
 240         results = self.rspec.xpath(query)
 241         if results == None:
 242             query = "./network[@name='%s']/link[@endpoints = '%s %s']" % (network, toid, fromid)
 243             results = self.rspec.xpath(query)
 244         return results
 245
 246     def query_vlinks(self, endpoints, network=None):
 247         if network == None:
 248             network = self.network
 249         query = "./network[@name='%s']//vlink[@endpoints = '%s']" % (network, endpoints)
 250         results = self.rspec.xpath(query)
 251         return results
 252
 253
 254     def add_vlink(self, fromhost, tohost, kbps, network=None):
 255         if network == None:
 256             network = self.network
 257         fromnode = self.get_node_element(fromhost, network)
 258         tonode = self.get_node_element(tohost, network)
 259         links = self.query_links(fromnode, tonode, network)
 260
 261         for link in links:
 262             vlink = etree.SubElement(link, "vlink")
 263             fromid = fromnode.get("id")
 264             toid = tonode.get("id")
 265             vlink.set("endpoints", "%s %s" % (fromid, toid))
 266             self.add_attribute(vlink, "kbps", kbps)
 267
 268
 269     def remove_vlink(self, endpoints, network=None):
 270         if network == None:
 271             network = self.network
 272         vlinks = self.query_vlinks(endpoints, network)
 273         for vlink in vlinks:
 274             vlink.getparent().remove(vlink)
 275
 276     def toxml(self):
 277         return etree.tostring(self.rspec, pretty_print=True,
 278                               xml_declaration=True)
 279
 280     def __str__(self):
 281         return self.toxml()
 282
 283     def save(self, filename):
 284         f = open(filename, "w")
 285         f.write(self.toxml())
 286         f.close()
 287
 288
 289 class Commands:
 290     def __init__(self, usage, description, epilog=None):
 291         self.parser = OptionParser(usage=usage, description=description,
 292                                    epilog=epilog)
 293         self.parser.add_option("-i", "", dest="infile", metavar="FILE",
 294                                help="read RSpec from FILE (default is stdin)")
 295         self.parser.add_option("-o", "", dest="outfile", metavar="FILE",
 296                                help="write output to FILE (default is stdout)")
 297         self.nodefile = False
 298         self.attributes = {}
 299
 300     def add_nodefile_option(self):
 301         self.nodefile = True
 302         self.parser.add_option("-n", "", dest="nodefile",
 303                                metavar="FILE",
 304                                help="read node list from FILE"),
 305
 306     def add_show_attributes_option(self):
 307         self.parser.add_option("-s", "--show-attributes", action="store_true",
 308                                dest="showatt", default=False,
 309                                help="show sliver attributes")
 310
 311     def add_attribute_options(self):
 312         self.parser.add_option("", "--capabilities", action="append",
 313                                metavar="<cap1,cap2,...>",
 314                                help="Vserver bcapabilities")
 315         self.parser.add_option("", "--codemux", action="append",
 316                                metavar="<host,local-port>",
 317                                help="Demux HTTP between slices using " +
 318                                "localhost ports")
 319         self.parser.add_option("", "--cpu-pct", action="append",
 320                                metavar="<num>",
 321                                help="Reserved CPU percent (e.g., 25)")
 322         self.parser.add_option("", "--cpu-share", action="append",
 323                                metavar="<num>",
 324                                help="Number of CPU shares (e.g., 5)")
 325         self.parser.add_option("", "--delegations",
 326                                metavar="<slice1,slice2,...>", action="append",
 327                                help="List of slices with delegation authority")
 328         self.parser.add_option("", "--disk-max",
 329                                metavar="<num>", action="append",
 330                                help="Disk quota (1k disk blocks)")
 331         self.parser.add_option("", "--initscript",
 332                                metavar="<name>", action="append",
 333                                help="Slice initialization script (e.g., stork)")
 334         self.parser.add_option("", "--ip-addresses", action="append",
 335                                metavar="<IP addr>",
 336                                help="Add an IP address to a sliver")
 337         self.parser.add_option("", "--net-i2-max-kbyte",
 338                                metavar="<KBytes>", action="append",
 339                                help="Maximum daily network Tx limit " +
 340                                "to I2 hosts.")
 341         self.parser.add_option("", "--net-i2-max-rate",
 342                                metavar="<Kbps>", action="append",
 343                                help="Maximum bandwidth over I2 routes")
 344         self.parser.add_option("", "--net-i2-min-rate",
 345                                metavar="<Kbps>", action="append",
 346                                help="Minimum bandwidth over I2 routes")
 347         self.parser.add_option("", "--net-i2-share",
 348                                metavar="<num>", action="append",
 349                                help="Number of bandwidth shares over I2 routes")
 350         self.parser.add_option("", "--net-i2-thresh-kbyte",
 351                                metavar="<KBytes>", action="append",
 352                                help="Limit sent to I2 hosts before warning, " +
 353                                "throttling")
 354         self.parser.add_option("", "--net-max-kbyte",
 355                                metavar="<KBytes>", action="append",
 356                                help="Maximum daily network Tx limit " +
 357                                "to non-I2 hosts.")
 358         self.parser.add_option("", "--net-max-rate",
 359                                metavar="<Kbps>", action="append",
 360                                help="Maximum bandwidth over non-I2 routes")
 361         self.parser.add_option("", "--net-min-rate",
 362                                metavar="<Kbps>", action="append",
 363                                help="Minimum bandwidth over non-I2 routes")
 364         self.parser.add_option("", "--net-share",
 365                                metavar="<num>", action="append",
 366                                help="Number of bandwidth shares over non-I2 " +
 367                                "routes")
 368         self.parser.add_option("", "--net-thresh-kbyte",
 369                                metavar="<KBytes>", action="append",
 370                                help="Limit sent to non-I2 hosts before " +
 371                                "warning, throttling")
 372         self.parser.add_option("", "--vsys",
 373                                metavar="<name>", action="append",
 374                                help="Vsys script (e.g., fd_fusemount)")
 375         self.parser.add_option("", "--vsys-vnet",
 376                                metavar="<IP network>", action="append",
 377                                help="Allocate a virtual private network")
 378
 379     def get_attribute_dict(self):
 380         attrlist = ['capabilities','codemux','cpu_pct','cpu_share',
 381                     'delegations','disk_max','initscript','ip_addresses',
 382                     'net_i2_max_kbyte','net_i2_max_rate','net_i2_min_rate',
 383                     'net_i2_share','net_i2_thresh_kbyte',
 384                     'net_max_kbyte','net_max_rate','net_min_rate',
 385                     'net_share','net_thresh_kbyte',
 386                     'vsys','vsys_vnet']
 387         attrdict = {}
 388         for attr in attrlist:
 389             value = getattr(self.opts, attr, None)
 390             if value is not None:
 391                 attrdict[attr] = value
 392         return attrdict
 393
 394     def prep(self):
 395         (self.opts, self.args) = self.parser.parse_args()
 396
 397         if self.opts.infile:
 398             sys.stdin = open(self.opts.infile, "r")
 399         xml = sys.stdin.read()
 400         self.rspec = RSpec(xml)
 401
 402         if self.nodefile:
 403             if self.opts.nodefile:
 404                 f = open(self.opts.nodefile, "r")
 405                 self.nodes = f.read().split()
 406                 f.close()
 407             else:
 408                 self.nodes = self.args
 409
 410         if self.opts.outfile:
 411             sys.stdout = open(self.opts.outfile, "w")
 412
 413
 414
 415
 416
 417
 418