sfa/client/sfascan.py

   1 from __future__ import print_function
   2
   3 import sys, os.path
   4 import pickle
   5 import time
   6 import socket
   7 import traceback
   8 from urlparse import urlparse
   9
  10 try:
  11     import pygraphviz
  12 except:
  13     print('Warning, could not import pygraphviz, test mode only')
  14
  15 from optparse import OptionParser
  16
  17 from sfa.client.return_value import ReturnValue
  18 from sfa.client.sfi import Sfi
  19 from sfa.util.sfalogging import logger, DEBUG
  20 from sfa.client.sfaserverproxy import SfaServerProxy
  21
  22 def url_hostname_port (url):
  23     if url.find("://")<0:
  24         url="http://"+url
  25     parsed_url=urlparse(url)
  26     # 0(scheme) returns protocol
  27     default_port='80'
  28     if parsed_url[0]=='https': default_port='443'
  29     # 1(netloc) returns the hostname+port part
  30     parts=parsed_url[1].split(":")
  31     # just a hostname
  32     if len(parts)==1:
  33         return (url,parts[0],default_port)
  34     else:
  35         return (url,parts[0],parts[1])
  36
  37 ### a very simple cache mechanism so that successive runs (see make)
  38 ### will go *much* faster
  39 ### assuming everything is sequential, as simple as it gets
  40 ### { url -> (timestamp,version)}
  41 class VersionCache:
  42     # default expiration period is 1h
  43     def __init__ (self, filename=None, expires=60*60):
  44         # default is to store cache in the same dir as argv[0]
  45         if filename is None:
  46             filename=os.path.join(os.path.dirname(sys.argv[0]),"sfascan-version-cache.pickle")
  47         self.filename=filename
  48         self.expires=expires
  49         self.url2version={}
  50         self.load()
  51
  52     def load (self):
  53         try:
  54             infile=file(self.filename,'r')
  55             self.url2version=pickle.load(infile)
  56             infile.close()
  57         except:
  58             logger.debug("Cannot load version cache, restarting from scratch")
  59             self.url2version = {}
  60         logger.debug("loaded version cache with %d entries %s"%(len(self.url2version),self.url2version.keys()))
  61
  62     def save (self):
  63         try:
  64             outfile=file(self.filename,'w')
  65             pickle.dump(self.url2version,outfile)
  66             outfile.close()
  67         except:
  68             logger.log_exc ("Cannot save version cache into %s"%self.filename)
  69     def clean (self):
  70         try:
  71             retcod=os.unlink(self.filename)
  72             logger.info("Cleaned up version cache %s, retcod=%d"%(self.filename,retcod))
  73         except:
  74             logger.info ("Could not unlink version cache %s"%self.filename)
  75
  76     def show (self):
  77         entries=len(self.url2version)
  78         print("version cache from file %s has %d entries"%(self.filename,entries))
  79         key_values=self.url2version.items()
  80         def old_first (kv1,kv2): return int(kv1[1][0]-kv2[1][0])
  81         key_values.sort(old_first)
  82         for key_value in key_values:
  83             (url,tuple) = key_value
  84             (timestamp,version) = tuple
  85             how_old = time.time()-timestamp
  86             if how_old<=self.expires:
  87                 print(url,"-- %d seconds ago"%how_old)
  88             else:
  89                 print("OUTDATED",url,"(%d seconds ago, expires=%d)"%(how_old,self.expires))
  90
  91     # turns out we might have trailing slashes or not
  92     def normalize (self, url):
  93         return url.strip("/")
  94
  95     def set (self,url,version):
  96         url=self.normalize(url)
  97         self.url2version[url]=( time.time(), version)
  98     def get (self,url):
  99         url=self.normalize(url)
 100         try:
 101             (timestamp,version)=self.url2version[url]
 102             how_old = time.time()-timestamp
 103             if how_old<=self.expires: return version
 104             else: return None
 105         except:
 106             return None
 107
 108 ###
 109 # non-existing hostnames happen...
 110 # for better perfs we cache the result of gethostbyname too
 111 class Interface:
 112
 113     def __init__ (self,url,mentioned_in=None,verbose=False):
 114         self._url=url
 115         self.verbose=verbose
 116         cache=VersionCache()
 117         key="interface:%s"%url
 118         try:
 119             (self._url,self.hostname,self.port)=url_hostname_port(url)
 120             # look for ip in the cache
 121             tuple=cache.get(key)
 122             if tuple:
 123                 (self.hostname, self.ip, self.port) = tuple
 124             else:
 125                 self.ip=socket.gethostbyname(self.hostname)
 126         except:
 127             msg="can't resolve hostname %s\n\tfound in url %s"%(self.hostname,self._url)
 128             if mentioned_in:
 129                 msg += "\n\t(mentioned at %s)"%mentioned_in
 130             logger.warning (msg)
 131             self.hostname="unknown"
 132             self.ip='0.0.0.0'
 133             self.port="???"
 134
 135         cache.set(key, (self.hostname, self.ip, self.port,) )
 136         cache.save()
 137         self.probed=False
 138
 139         # mark unknown interfaces as probed to avoid unnecessary attempts
 140         if self.hostname=='unknown':
 141             # don't really try it
 142             self.probed=True
 143             self._version={}
 144
 145
 146     def url(self):
 147         return self._url
 148
 149     # this is used as a key for creating graph nodes and to avoid duplicates
 150     def uid (self):
 151         return "%s:%s"%(self.ip,self.port)
 152
 153     # connect to server and trigger GetVersion
 154     def get_version(self):
 155         ### if we already know the answer:
 156         if self.probed:
 157             return self._version
 158         ### otherwise let's look in the cache file
 159         logger.debug("searching in version cache %s"%self.url())
 160         cached_version = VersionCache().get(self.url())
 161         if cached_version is not None:
 162             logger.info("Retrieved version info from cache %s"%self.url())
 163             return cached_version
 164         ### otherwise let's do the hard work
 165         # dummy to meet Sfi's expectations for its 'options' field
 166         class DummyOptions:
 167             pass
 168         options=DummyOptions()
 169         options.verbose=self.verbose
 170         options.timeout=10
 171         try:
 172             client=Sfi(options)
 173             client.read_config()
 174             client.bootstrap()
 175             key_file = client.private_key
 176             cert_file = client.my_gid
 177             logger.debug("using key %s & cert %s"%(key_file,cert_file))
 178             url=self.url()
 179             logger.info('issuing GetVersion at %s'%url)
 180             # setting timeout here seems to get the call to fail - even though the response time is fast
 181             #server=SfaServerProxy(url, key_file, cert_file, verbose=self.verbose, timeout=options.timeout)
 182             server=SfaServerProxy(url, key_file, cert_file, verbose=self.verbose)
 183             self._version=ReturnValue.get_value(server.GetVersion())
 184         except:
 185             logger.log_exc("failed to get version")
 186             self._version={}
 187         # so that next run from this process will find out
 188         self.probed=True
 189         # store in version cache so next processes will remember for an hour
 190         cache=VersionCache()
 191         cache.set(self.url(),self._version)
 192         cache.save()
 193         logger.debug("Saved version for url=%s in version cache"%self.url())
 194         # that's our result
 195         return self._version
 196
 197     @staticmethod
 198     def multi_lines_label(*lines):
 199         result='<<TABLE BORDER="0" CELLBORDER="0"><TR><TD>' + \
 200             '</TD></TR><TR><TD>'.join(lines) + \
 201             '</TD></TR></TABLE>>'
 202         return result
 203
 204     # default is for when we can't determine the type of the service
 205     # typically the server is down, or we can't authenticate, or it's too old code
 206     shapes = {"registry": "diamond", "slicemgr":"ellipse", "aggregate":"box", 'default':'plaintext'}
 207     abbrevs = {"registry": "REG", "slicemgr":"SA", "aggregate":"AM", 'default':'[unknown interface]'}
 208
 209     # return a dictionary that translates into the node's attr
 210     def get_layout (self):
 211         layout={}
 212         ### retrieve cached GetVersion
 213         version=self.get_version()
 214         # set the href; xxx would make sense to try and 'guess' the web URL, not the API's one...
 215         layout['href']=self.url()
 216         ### set html-style label
 217         ### see http://www.graphviz.org/doc/info/shapes.html#html
 218         # if empty the service is unreachable
 219         if not version:
 220             label="offline"
 221         else:
 222             label=''
 223             try: abbrev=Interface.abbrevs[version['interface']]
 224             except: abbrev=Interface.abbrevs['default']
 225             label += abbrev
 226             if 'hrn' in version: label += " %s"%version['hrn']
 227             else:                label += "[no hrn]"
 228             if 'code_tag' in version:
 229                 label += " %s"%version['code_tag']
 230             if 'testbed' in version:
 231                 label += " (%s)"%version['testbed']
 232         layout['label']=Interface.multi_lines_label(self.url(),label)
 233         ### set shape
 234         try: shape=Interface.shapes[version['interface']]
 235         except: shape=Interface.shapes['default']
 236         layout['shape']=shape
 237         ### fill color to outline wrongly configured or unreachable bodies
 238         # as of sfa-2.0 registry doesn't have 'sfa' not 'geni_api', but have peers
 239         # slicemgr and aggregate have 'geni_api' and 'sfa'
 240         if 'geni_api' not in version and 'peers' not in version:
 241             layout['style']='filled'
 242             layout['fillcolor']='gray'
 243         return layout
 244
 245 class Scanner:
 246
 247     # provide the entry points (a list of interfaces)
 248     def __init__ (self, left_to_right=False, verbose=False):
 249         self.verbose=verbose
 250         self.left_to_right=left_to_right
 251
 252     def graph (self,entry_points):
 253         graph=pygraphviz.AGraph(directed=True)
 254         if self.left_to_right:
 255             graph.graph_attr['rankdir']='LR'
 256         self.scan(entry_points,graph)
 257         return graph
 258
 259     # scan from the given interfaces as entry points
 260     def scan(self,interfaces,graph):
 261         if not isinstance(interfaces,list):
 262             interfaces=[interfaces]
 263
 264         # remember node to interface mapping
 265         node2interface={}
 266         # add entry points right away using the interface uid's as a key
 267         to_scan=interfaces
 268         for i in interfaces:
 269             graph.add_node(i.uid())
 270             node2interface[graph.get_node(i.uid())]=i
 271         scanned=[]
 272         # keep on looping until we reach a fixed point
 273         # don't worry about abels and shapes that will get fixed later on
 274         while to_scan:
 275             for interface in to_scan:
 276                 # performing xmlrpc call
 277                 logger.info("retrieving/fetching version at interface %s"%interface.url())
 278                 version=interface.get_version()
 279                 if not version:
 280                     logger.info("<EMPTY GetVersion(); offline or cannot authenticate>")
 281                 else:
 282                     for (k,v) in version.iteritems():
 283                         if not isinstance(v,dict):
 284                             logger.debug("\r\t%s:%s"%(k,v))
 285                         else:
 286                             logger.debug(k)
 287                             for (k1,v1) in v.iteritems():
 288                                 logger.debug("\r\t\t%s:%s"%(k1,v1))
 289                 # proceed with neighbours
 290                 if 'peers' in version:
 291                     for (next_name,next_url) in version['peers'].iteritems():
 292                         next_interface=Interface(next_url,mentioned_in=interface.url())
 293                         # locate or create node in graph
 294                         try:
 295                             # if found, we're good with this one
 296                             next_node=graph.get_node(next_interface.uid())
 297                         except:
 298                             # otherwise, let's move on with it
 299                             graph.add_node(next_interface.uid())
 300                             next_node=graph.get_node(next_interface.uid())
 301                             node2interface[next_node]=next_interface
 302                             to_scan.append(next_interface)
 303                         graph.add_edge(interface.uid(),next_interface.uid())
 304                 scanned.append(interface)
 305                 to_scan.remove(interface)
 306             # we've scanned the whole graph, let's get the labels and shapes right
 307             for node in graph.nodes():
 308                 interface=node2interface.get(node,None)
 309                 if interface:
 310                     for (k,v) in interface.get_layout().iteritems():
 311                         node.attr[k]=v
 312                 else:
 313                     logger.error("MISSED interface with node %s"%node)
 314
 315
 316 class SfaScan:
 317
 318     default_outfiles=['sfa.png','sfa.svg','sfa.dot']
 319
 320     def main(self):
 321         usage="%prog [options] url-entry-point(s)"
 322         parser=OptionParser(usage=usage)
 323         parser.add_option("-d", "--dir", dest="sfi_dir",
 324                           help="config & working directory - default is " + Sfi.default_sfi_dir(),
 325                           metavar="PATH", default=Sfi.default_sfi_dir())
 326         parser.add_option("-o","--output",action='append',dest='outfiles',default=[],
 327                           help="output filenames (cumulative) - defaults are %r"%SfaScan.default_outfiles)
 328         parser.add_option("-l","--left-to-right",action="store_true",dest="left_to_right",default=False,
 329                           help="instead of top-to-bottom")
 330         parser.add_option("-v", "--verbose", action="count", dest="verbose", default=0,
 331                           help="verbose - can be repeated for more verbosity")
 332         parser.add_option("-c", "--clean-cache",action='store_true',
 333                           dest='clean_cache',default=False,
 334                           help='clean/trash version cache and exit')
 335         parser.add_option("-s","--show-cache",action='store_true',
 336                           dest='show_cache',default=False,
 337                           help='show/display version cache')
 338
 339         (options,args)=parser.parse_args()
 340         logger.enable_console()
 341         # apply current verbosity to logger
 342         logger.setLevelFromOptVerbose(options.verbose)
 343         # figure if we need to be verbose for these local classes that only have a bool flag
 344         bool_verbose=logger.getBoolVerboseFromOpt(options.verbose)
 345
 346         if options.show_cache:
 347             VersionCache().show()
 348             sys.exit(0)
 349         if options.clean_cache:
 350             VersionCache().clean()
 351             sys.exit(0)
 352         if not args:
 353             parser.print_help()
 354             sys.exit(1)
 355
 356         if not options.outfiles:
 357             options.outfiles=SfaScan.default_outfiles
 358         scanner=Scanner(left_to_right=options.left_to_right, verbose=bool_verbose)
 359         entries = [ Interface(entry,mentioned_in="command line") for entry in args ]
 360         try:
 361             g=scanner.graph(entries)
 362             logger.info("creating layout")
 363             g.layout(prog='dot')
 364             for outfile in options.outfiles:
 365                 logger.info("drawing in %s"%outfile)
 366                 g.draw(outfile)
 367             logger.info("done")
 368         # test mode when pygraphviz is not available
 369         except:
 370             entry=entries[0]
 371             print("GetVersion at %s returned %s"%(entry.url(),entry.get_version()))
 372