sfa/client/sfascan.py

   1 import sys, os.path
   2 import pickle
   3 import time
   4 import socket
   5 import traceback
   6 from urlparse import urlparse
   7
   8 try:
   9     import pygraphviz
  10 except:
  11     print 'Warning, could not import pygraphviz, test mode only'
  12
  13 from optparse import OptionParser
  14
  15 from sfa.client.sfi import Sfi
  16 from sfa.util.sfalogging import logger, DEBUG
  17 import sfa.client.sfaprotocol as sfaprotocol
  18
  19 def url_hostname_port (url):
  20     if url.find("://")<0:
  21         url="http://"+url
  22     parsed_url=urlparse(url)
  23     # 0(scheme) returns protocol
  24     default_port='80'
  25     if parsed_url[0]=='https': default_port='443'
  26     # 1(netloc) returns the hostname+port part
  27     parts=parsed_url[1].split(":")
  28     # just a hostname
  29     if len(parts)==1:
  30         return (url,parts[0],default_port)
  31     else:
  32         return (url,parts[0],parts[1])
  33
  34 ### a very simple cache mechanism so that successive runs (see make)
  35 ### will go *much* faster
  36 ### assuming everything is sequential, as simple as it gets
  37 ### { url -> (timestamp,version)}
  38 class VersionCache:
  39     def __init__ (self, filename=None, expires=60*60):
  40         # default is to store cache in the same dir as argv[0]
  41         if filename is None:
  42             filename=os.path.join(os.path.dirname(sys.argv[0]),"sfascan-version-cache.pickle")
  43         self.filename=filename
  44         self.expires=expires
  45         self.url2version={}
  46         self.load()
  47
  48     def load (self):
  49         try:
  50             infile=file(self.filename,'r')
  51             self.url2version=pickle.load(infile)
  52             infile.close()
  53         except:
  54             logger.debug("Cannot load version cache, restarting from scratch")
  55             self.url2version = {}
  56         logger.debug("loaded version cache with %d entries %s"%(len(self.url2version),self.url2version.keys()))
  57
  58     def save (self):
  59         try:
  60             outfile=file(self.filename,'w')
  61             pickle.dump(self.url2version,outfile)
  62             outfile.close()
  63         except:
  64             logger.log_exc ("Cannot save version cache into %s"%self.filename)
  65     def clean (self):
  66         try:
  67             retcod=os.unlink(self.filename)
  68             logger.info("Cleaned up version cache %s, retcod=%d"%(self.filename,retcod))
  69         except:
  70             logger.info ("Could not unlink version cache %s"%self.filename)
  71
  72     def show (self):
  73         entries=len(self.url2version)
  74         print "version cache from file %s has %d entries"%(self.filename,entries)
  75         key_values=self.url2version.items()
  76         def old_first (kv1,kv2): return int(kv1[1][0]-kv2[1][0])
  77         key_values.sort(old_first)
  78         for key_value in key_values:
  79             (url,tuple) = key_value
  80             (timestamp,version) = tuple
  81             how_old = time.time()-timestamp
  82             if how_old<=self.expires:
  83                 print url,"-- %d seconds ago"%how_old
  84             else:
  85                 print "OUTDATED",url,"(%d seconds ago, expires=%d)"%(how_old,self.expires)
  86
  87     # turns out we might have trailing slashes or not
  88     def normalize (self, url):
  89         return url.strip("/")
  90
  91     def set (self,url,version):
  92         url=self.normalize(url)
  93         self.url2version[url]=( time.time(), version)
  94     def get (self,url):
  95         url=self.normalize(url)
  96         try:
  97             (timestamp,version)=self.url2version[url]
  98             how_old = time.time()-timestamp
  99             if how_old<=self.expires: return version
 100             else: return None
 101         except:
 102             return None
 103
 104 ###
 105 class Interface:
 106
 107     def __init__ (self,url,verbose=False):
 108         self._url=url
 109         self.verbose=verbose
 110         try:
 111             (self._url,self.hostname,self.port)=url_hostname_port(url)
 112             self.ip=socket.gethostbyname(self.hostname)
 113             self.probed=False
 114         except:
 115             self.hostname="unknown"
 116             self.ip='0.0.0.0'
 117             self.port="???"
 118             # don't really try it
 119             self.probed=True
 120             self._version={}
 121
 122     def url(self):
 123         return self._url
 124
 125     # this is used as a key for creating graph nodes and to avoid duplicates
 126     def uid (self):
 127         return "%s:%s"%(self.ip,self.port)
 128
 129     # connect to server and trigger GetVersion
 130     def get_version(self):
 131         ### if we already know the answer:
 132         if self.probed:
 133             return self._version
 134         ### otherwise let's look in the cache file
 135         logger.debug("searching in version cache %s"%self.url())
 136         cached_version = VersionCache().get(self.url())
 137         if cached_version is not None:
 138             logger.info("Retrieved version info from cache")
 139             return cached_version
 140         ### otherwise let's do the hard work
 141         # dummy to meet Sfi's expectations for its 'options' field
 142         class DummyOptions:
 143             pass
 144         options=DummyOptions()
 145         options.verbose=self.verbose
 146         options.timeout=10
 147         try:
 148             client=Sfi(options)
 149             client.read_config()
 150             key_file = client.get_key_file()
 151             cert_file = client.get_cert_file(key_file)
 152             logger.debug("using key %s & cert %s"%(key_file,cert_file))
 153             url=self.url()
 154             logger.info('issuing GetVersion at %s'%url)
 155             # setting timeout here seems to get the call to fail - even though the response time is fast
 156             #server=sfaprotocol.server_proxy(url, key_file, cert_file, verbose=self.verbose, timeout=options.timeout)
 157             server=sfaprotocol.server_proxy(url, key_file, cert_file, verbose=self.verbose)
 158             self._version=server.GetVersion()
 159         except:
 160             logger.log_exc("failed to get version")
 161             self._version={}
 162         # so that next run from this process will find out
 163         self.probed=True
 164         # store in version cache so next processes will remember for an hour
 165         cache=VersionCache()
 166         cache.set(self.url(),self._version)
 167         cache.save()
 168         logger.debug("Saved version for url=%s in version cache"%self.url())
 169         # that's our result
 170         return self._version
 171
 172     @staticmethod
 173     def multi_lines_label(*lines):
 174         result='<<TABLE BORDER="0" CELLBORDER="0"><TR><TD>' + \
 175             '</TD></TR><TR><TD>'.join(lines) + \
 176             '</TD></TR></TABLE>>'
 177         return result
 178
 179     # default is for when we can't determine the type of the service
 180     # typically the server is down, or we can't authenticate, or it's too old code
 181     shapes = {"registry": "diamond", "slicemgr":"ellipse", "aggregate":"box", 'default':'plaintext'}
 182     abbrevs = {"registry": "REG", "slicemgr":"SA", "aggregate":"AM", 'default':'[unknown interface]'}
 183
 184     # return a dictionary that translates into the node's attr
 185     def get_layout (self):
 186         layout={}
 187         ### retrieve cached GetVersion
 188         version=self.get_version()
 189         # set the href; xxx would make sense to try and 'guess' the web URL, not the API's one...
 190         layout['href']=self.url()
 191         ### set html-style label
 192         ### see http://www.graphviz.org/doc/info/shapes.html#html
 193         # if empty the service is unreachable
 194         if not version:
 195             label="offline"
 196         else:
 197             label=''
 198             try: abbrev=Interface.abbrevs[version['interface']]
 199             except: abbrev=Interface.abbrevs['default']
 200             label += abbrev
 201             if 'hrn' in version: label += " %s"%version['hrn']
 202             else:                label += "[no hrn]"
 203             if 'code_tag' in version:
 204                 label += " %s"%version['code_tag']
 205             if 'testbed' in version:
 206                 label += " (%s)"%version['testbed']
 207         layout['label']=Interface.multi_lines_label(self.url(),label)
 208         ### set shape
 209         try: shape=Interface.shapes[version['interface']]
 210         except: shape=Interface.shapes['default']
 211         layout['shape']=shape
 212         ### fill color to outline wrongly configured bodies
 213         if 'geni_api' not in version and 'sfa' not in version:
 214             layout['style']='filled'
 215             layout['fillcolor']='gray'
 216         return layout
 217
 218 class Scanner:
 219
 220     # provide the entry points (a list of interfaces)
 221     def __init__ (self, left_to_right=False, verbose=False):
 222         self.verbose=verbose
 223         self.left_to_right=left_to_right
 224
 225     def graph (self,entry_points):
 226         graph=pygraphviz.AGraph(directed=True)
 227         if self.left_to_right:
 228             graph.graph_attr['rankdir']='LR'
 229         self.scan(entry_points,graph)
 230         return graph
 231
 232     # scan from the given interfaces as entry points
 233     def scan(self,interfaces,graph):
 234         if not isinstance(interfaces,list):
 235             interfaces=[interfaces]
 236
 237         # remember node to interface mapping
 238         node2interface={}
 239         # add entry points right away using the interface uid's as a key
 240         to_scan=interfaces
 241         for i in interfaces:
 242             graph.add_node(i.uid())
 243             node2interface[graph.get_node(i.uid())]=i
 244         scanned=[]
 245         # keep on looping until we reach a fixed point
 246         # don't worry about abels and shapes that will get fixed later on
 247         while to_scan:
 248             for interface in to_scan:
 249                 # performing xmlrpc call
 250                 logger.info("retrieving/fetching version at interface %s"%interface.url())
 251                 version=interface.get_version()
 252                 if not version:
 253                     logger.info("<EMPTY GetVersion(); offline or cannot authenticate>")
 254                 else:
 255                     for (k,v) in version.iteritems():
 256                         if not isinstance(v,dict):
 257                             logger.debug("\r\t%s:%s"%(k,v))
 258                         else:
 259                             logger.debug(k)
 260                             for (k1,v1) in v.iteritems():
 261                                 logger.debug("\r\t\t%s:%s"%(k1,v1))
 262                 # 'geni_api' is expected if the call succeeded at all
 263                 # 'peers' is needed as well as AMs typically don't have peers
 264                 if 'geni_api' in version and 'peers' in version:
 265                     # proceed with neighbours
 266                     for (next_name,next_url) in version['peers'].iteritems():
 267                         next_interface=Interface(next_url)
 268                         # locate or create node in graph
 269                         try:
 270                             # if found, we're good with this one
 271                             next_node=graph.get_node(next_interface.uid())
 272                         except:
 273                             # otherwise, let's move on with it
 274                             graph.add_node(next_interface.uid())
 275                             next_node=graph.get_node(next_interface.uid())
 276                             node2interface[next_node]=next_interface
 277                             to_scan.append(next_interface)
 278                         graph.add_edge(interface.uid(),next_interface.uid())
 279                 scanned.append(interface)
 280                 to_scan.remove(interface)
 281             # we've scanned the whole graph, let's get the labels and shapes right
 282             for node in graph.nodes():
 283                 interface=node2interface.get(node,None)
 284                 if interface:
 285                     for (k,v) in interface.get_layout().iteritems():
 286                         node.attr[k]=v
 287                 else:
 288                     logger.error("MISSED interface with node %s"%node)
 289
 290
 291 class SfaScan:
 292
 293     default_outfiles=['sfa.png','sfa.svg','sfa.dot']
 294
 295     def main(self):
 296         usage="%prog [options] url-entry-point(s)"
 297         parser=OptionParser(usage=usage)
 298         parser.add_option("-d", "--dir", dest="sfi_dir",
 299                           help="config & working directory - default is " + Sfi.default_sfi_dir(),
 300                           metavar="PATH", default=Sfi.default_sfi_dir())
 301         parser.add_option("-o","--output",action='append',dest='outfiles',default=[],
 302                           help="output filenames (cumulative) - defaults are %r"%SfaScan.default_outfiles)
 303         parser.add_option("-l","--left-to-right",action="store_true",dest="left_to_right",default=False,
 304                           help="instead of top-to-bottom")
 305         parser.add_option("-v", "--verbose", action="count", dest="verbose", default=0,
 306                           help="verbose - can be repeated for more verbosity")
 307         parser.add_option("-c", "--clean-cache",action='store_true',
 308                           dest='clean_cache',default=False,
 309                           help='clean/trash version cache and exit')
 310         parser.add_option("-s","--show-cache",action='store_true',
 311                           dest='show_cache',default=False,
 312                           help='show/display version cache')
 313
 314         (options,args)=parser.parse_args()
 315         logger.enable_console()
 316         # apply current verbosity to logger
 317         logger.setLevelFromOptVerbose(options.verbose)
 318         # figure if we need to be verbose for these local classes that only have a bool flag
 319         bool_verbose=logger.getBoolVerboseFromOpt(options.verbose)
 320
 321         if options.show_cache:
 322             VersionCache().show()
 323             sys.exit(0)
 324         if options.clean_cache:
 325             VersionCache().clean()
 326             sys.exit(0)
 327         if not args:
 328             parser.print_help()
 329             sys.exit(1)
 330
 331         if not options.outfiles:
 332             options.outfiles=SfaScan.default_outfiles
 333         scanner=Scanner(left_to_right=options.left_to_right, verbose=bool_verbose)
 334         entries = [ Interface(entry) for entry in args ]
 335         try:
 336             g=scanner.graph(entries)
 337             logger.info("creating layout")
 338             g.layout(prog='dot')
 339             for outfile in options.outfiles:
 340                 logger.info("drawing in %s"%outfile)
 341                 g.draw(outfile)
 342             logger.info("done")
 343         # test mode when pygraphviz is not available
 344         except:
 345             entry=entries[0]
 346             print "GetVersion at %s returned %s"%(entry.url(),entry.get_version())
 347