monitor/wrapper/plc.py

   1 #
   2 # plc.py
   3 #
   4 # Helper functions that minipulate the PLC api.
   5 #
   6 # Faiyaz Ahmed <faiyaza@cs.princeton.edu
   7 #
   8 # $Id: plc.py,v 1.18 2007/08/29 17:26:50 soltesz Exp $
   9 #
  10
  11 import xml, xmlrpclib
  12 import logging
  13 import time
  14 import traceback
  15 from datetime import datetime
  16
  17 # note: this needs to be consistent with the value in PLEWWW/planetlab/includes/plc_functions.php
  18 PENDING_CONSORTIUM_ID = 0
  19 # not used in monitor
  20 #APPROVED_CONSORTIUM_ID = 999999
  21
  22 try:
  23         from monitor import config
  24         debug = config.debug
  25         XMLRPC_SERVER=config.API_SERVER
  26 except:
  27         debug = False
  28         # NOTE: this host is used by default when there are no auth files.
  29         XMLRPC_SERVER="https://boot.planet-lab.org/PLCAPI/"
  30
  31 global_log_api = True
  32 logging.basicConfig(level=logging.DEBUG,
  33                     format='%(asctime)s %(levelname)s %(name)s : %(message)s',
  34                     datefmt='%s %Y-%m-%dT%H:%M:%S',
  35                     filename='/usr/share/monitor/myops-api-log.log',
  36                     filemode='a')
  37 apilog = logging.getLogger("api")
  38
  39 def log_api_call(name, *params):
  40     logstr = "%s(" %name
  41     for x in params:
  42         logstr += "%s," % x
  43     logstr = logstr[:-1] + ")"
  44     if global_log_api: apilog.debug(logstr)
  45
  46 logger = logging.getLogger("monitor")
  47
  48 class Auth:
  49         def __init__(self, username=None, password=None, **kwargs):
  50                 if 'session' in kwargs:
  51                         self.auth= { 'AuthMethod' : 'session',
  52                                         'session' : kwargs['session'] }
  53                 else:
  54                         if username==None and password==None:
  55                                 self.auth = {'AuthMethod': "anonymous"}
  56                         else:
  57                                 self.auth = {'Username' : username,
  58                                                         'AuthMethod' : 'password',
  59                                                         'AuthString' : password}
  60
  61
  62 # NOTE: by default, use anonymous access, but if auth files are
  63 #       configured, use them, with their auth definitions.
  64 auth = Auth()
  65 try:
  66         from monitor import config
  67         auth.auth = {'Username' : config.API_AUTH_USER,
  68                      'AuthMethod' : 'password',
  69                                  'AuthString' : config.API_AUTH_PASSWORD}
  70         auth.server = config.API_SERVER
  71 except:
  72         try:
  73                 import auth
  74                 auth.server = auth.plc
  75         except:
  76                 auth = Auth()
  77                 auth.server = XMLRPC_SERVER
  78
  79 global_error_count = 0
  80
  81 class PLC:
  82         def __init__(self, auth, url):
  83                 self.auth = auth
  84                 self.url = url
  85                 self.api = xmlrpclib.Server(self.url, verbose=False, allow_none=True)
  86
  87         def __getattr__(self, name):
  88                 method = getattr(self.api, name)
  89                 if method is None:
  90                         raise AssertionError("method does not exist")
  91
  92                 try:
  93                         def call_method(aut, *params):
  94                                 if global_log_api: log_api_call(name, *params)
  95                                 return method(aut, *params)
  96                         return lambda *params : call_method(self.auth, *params)
  97                         #return lambda *params : method(self.auth, *params)
  98                 except xmlrpclib.ProtocolError:
  99                         traceback.print_exc()
 100                         global_error_count += 1
 101                         if global_error_count >= 10:
 102                                 print "maximum error count exceeded; exiting..."
 103                                 sys.exit(1)
 104                         else:
 105                                 print "%s errors have occurred" % global_error_count
 106                         raise Exception("ProtocolError continuing")
 107
 108         def __repr__(self):
 109                 return self.api.__repr__()
 110
 111
 112 api = PLC(auth.auth, auth.server)
 113
 114
 115 def getAPI(url):
 116         return xmlrpclib.Server(url, verbose=False, allow_none=True)
 117
 118 def getNodeAPI(session):
 119         nodeauth = Auth(session=session)
 120         return PLC(nodeauth.auth, auth.server)
 121
 122 def getAuthAPI(url=None):
 123         if url:
 124                 return PLC(auth.auth, url)
 125         else:
 126                 return PLC(auth.auth, auth.server)
 127
 128 def getCachedAuthAPI():
 129         return CachedPLC(auth.auth, auth.server)
 130
 131 def getSessionAPI(session, server):
 132         nodeauth = Auth(session=session)
 133         return PLC(nodeauth.auth, server)
 134 def getUserAPI(username, password, server):
 135         auth = Auth(username,password)
 136         return PLC(auth.auth, server)
 137
 138 def getTechEmails(loginbase):
 139         """
 140                 For the given site, return all user email addresses that have the 'tech' role.
 141         """
 142         api = getAuthAPI()
 143         # get site details.
 144         s = api.GetSites(loginbase)[0]
 145         # get people at site
 146         p = api.GetPersons(s['person_ids'])
 147         # pull out those with the right role.
 148         emails = []
 149         for person in filter(lambda x: 'tech' in x['roles'], p):
 150                 if not isPersonExempt(person['email']):
 151                         emails.append(person['email'])
 152         #emails = [ person['email'] for person in filter(lambda x: 'tech' in x['roles'], p) ]
 153         return emails
 154
 155 def getPIEmails(loginbase):
 156         """
 157                 For the given site, return all user email addresses that have the 'tech' role.
 158         """
 159         api = getAuthAPI()
 160         # get site details.
 161         s = api.GetSites(loginbase)[0]
 162         # get people at site
 163         p = api.GetPersons(s['person_ids'])
 164         # pull out those with the right role.
 165         #emails = [ person['email'] for person in filter(lambda x: 'pi' in x['roles'], p) ]
 166         emails = []
 167         for person in filter(lambda x: 'pi' in x['roles'], p):
 168                 if not isPersonExempt(person['email']):
 169                         emails.append(person['email'])
 170         return emails
 171
 172 def getSliceUserEmails(loginbase):
 173         """
 174                 For the given site, return all user email addresses that have the 'tech' role.
 175         """
 176         api = getAuthAPI()
 177         # get site details.
 178         s = api.GetSites(loginbase)[0]
 179         # get people at site
 180         slices = api.GetSlices(s['slice_ids'])
 181         people = []
 182         for slice in slices:
 183                 people += api.GetPersons(slice['person_ids'])
 184         # pull out those with the right role.
 185         #emails = [ person['email'] for person in filter(lambda x: 'pi' in x['roles'], people) ]
 186
 187         emails = []
 188         for person in people:
 189                 if not isPersonExempt(person['email']):
 190                         emails.append(person['email'])
 191
 192         unique_emails = [ x for x in set(emails) ]
 193         return unique_emails
 194
 195 '''
 196 Returns list of nodes in dbg as reported by PLC
 197 '''
 198 def nodesDbg():
 199         dbgNodes = []
 200         api = xmlrpclib.Server(auth.server, verbose=False)
 201         anon = {'AuthMethod': "anonymous"}
 202         for node in api.GetNodes(anon, {"boot_state":"dbg"},["hostname"]):
 203                 dbgNodes.append(node['hostname'])
 204         logger.info("%s nodes in debug according to PLC." %len(dbgNodes))
 205         return dbgNodes
 206
 207
 208 '''
 209 Returns loginbase for given nodename
 210 '''
 211 def siteId(nodename):
 212         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 213         site_id = api.GetNodes (auth.auth, {"hostname": nodename}, ['site_id'])
 214         if len(site_id) == 1:
 215                 loginbase = api.GetSites (auth.auth, site_id[0], ["login_base"])
 216                 return loginbase[0]['login_base']
 217         else:
 218                 print "Not nodes returned!!!!"
 219
 220 '''
 221 Returns list of slices for a site.
 222 '''
 223 def slices(loginbase):
 224         siteslices = []
 225         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 226         sliceids = api.GetSites (auth.auth, {"login_base" : loginbase}, ["slice_ids"])[0]['slice_ids']
 227         for slice in api.GetSlices(auth.auth, {"slice_id" : sliceids}, ["name"]):
 228                 siteslices.append(slice['name'])
 229         return siteslices
 230
 231 '''
 232 Returns dict of PCU info of a given node.
 233 '''
 234 def getpcu(nodename):
 235         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 236         anon = {'AuthMethod': "anonymous"}
 237         try:
 238                 nodeinfo = api.GetNodes(auth.auth, {"hostname": nodename}, ["pcu_ids", "ports"])[0]
 239         except IndexError:
 240                 logger.info("Can not find node: %s" % nodename)
 241                 return False
 242         if nodeinfo['pcu_ids']:
 243                 print nodeinfo
 244                 sitepcu = api.GetPCUs(auth.auth, nodeinfo['pcu_ids'])[0]
 245                 print sitepcu
 246                 print nodeinfo["ports"]
 247                 sitepcu[nodename] = nodeinfo["ports"][0]
 248                 return sitepcu
 249         else:
 250                 logger.info("%s doesn't have PCU" % nodename)
 251                 return False
 252
 253 def GetPCUs(filter=None, fields=None):
 254         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 255         pcu_list = api.GetPCUs(auth.auth, filter, fields)
 256         return pcu_list
 257
 258 '''
 259 Returns all site nodes for site id (loginbase).
 260 '''
 261 def getSiteNodes(loginbase, fields=None):
 262         api = xmlrpclib.Server(auth.server, verbose=False)
 263         nodelist = []
 264         anon = {'AuthMethod': "anonymous"}
 265         try:
 266                 nodeids = api.GetSites(anon, {"login_base": loginbase}, fields)[0]['node_ids']
 267                 for node in api.GetNodes(anon, {"node_id": nodeids}, ['hostname']):
 268                         nodelist.append(node['hostname'])
 269         except Exception, exc:
 270                 logger.info("getSiteNodes:  %s" % exc)
 271                 print "getSiteNodes:  %s" % exc
 272         return nodelist
 273
 274
 275 def getPersons(filter=None, fields=None):
 276         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 277         persons = []
 278         try:
 279                 persons = api.GetPersons(auth.auth, filter, fields)
 280         except Exception, exc:
 281                 print "getPersons:  %s" % exc
 282                 logger.info("getPersons:  %s" % exc)
 283         return persons
 284
 285 def getSites(filter=None, fields=None):
 286         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 287         sites = []
 288         anon = {'AuthMethod': "anonymous"}
 289         try:
 290                 #sites = api.GetSites(anon, filter, fields)
 291                 sites = api.GetSites(auth.auth, filter, fields)
 292         except Exception, exc:
 293                 traceback.print_exc()
 294                 print "getSites:  %s" % exc
 295                 logger.info("getSites:  %s" % exc)
 296         return sites
 297
 298 def getSiteNodes2(loginbase):
 299         api = xmlrpclib.Server(auth.server, verbose=False)
 300         nodelist = []
 301         anon = {'AuthMethod': "anonymous"}
 302         try:
 303                 nodeids = api.GetSites(anon, {"login_base": loginbase})[0]['node_ids']
 304                 nodelist += getNodes({'node_id':nodeids})
 305         except Exception, exc:
 306                 logger.info("getSiteNodes2:  %s" % exc)
 307         return nodelist
 308
 309 def getNodeNetworks(filter=None):
 310         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 311         nodenetworks = api.GetInterfaces(auth.auth, filter, None)
 312         return nodenetworks
 313
 314 def getNodes(filter=None, fields=None):
 315         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 316         nodes = api.GetNodes(auth.auth, filter, fields)
 317                         #['boot_state', 'hostname',
 318                         #'site_id', 'date_created', 'node_id', 'version', 'interface_ids',
 319                         #'last_updated', 'peer_node_id', 'ssh_rsa_key' ])
 320         return nodes
 321
 322
 323 # Check if the site is a pending site that needs to be approved.
 324 def isPendingSite(loginbase):
 325         api = xmlrpclib.Server(auth.server, verbose=False)
 326         try:
 327                 site = api.GetSites(auth.auth, loginbase)[0]
 328         except Exception, exc:
 329                 logger.info("ERROR: No site %s" % loginbase)
 330                 return False
 331
 332         if not site['enabled'] and site['ext_consortium_id'] == PENDING_CONSORTIUM_ID:
 333                 return True
 334
 335         return False
 336
 337
 338 '''
 339 Sets boot state of a node.
 340 '''
 341 def nodeBootState(nodename, state):
 342         api = xmlrpclib.Server(auth.server, verbose=False)
 343         try:
 344                 return api.UpdateNode(auth.auth, nodename, {'boot_state': state})
 345         except Exception, exc:
 346                 logger.info("nodeBootState:  %s" % exc)
 347
 348 def updateNodeKey(nodename, key):
 349         api = xmlrpclib.Server(auth.server, verbose=False)
 350         try:
 351                 return api.UpdateNode(auth.auth, nodename, {'key': key})
 352         except Exception, exc:
 353                 logger.info("updateNodeKey:  %s" % exc)
 354
 355 '''
 356 Sends Ping Of Death to node.
 357 '''
 358 def nodePOD(nodename):
 359         api = xmlrpclib.Server(auth.server, verbose=False)
 360         logger.info("Sending POD to %s" % nodename)
 361         try:
 362                 if not debug:
 363                         return api.RebootNode(auth.auth, nodename)
 364         except Exception, exc:
 365                         logger.info("nodePOD:  %s" % exc)
 366
 367 '''
 368 Freeze all site slices.
 369 '''
 370 def suspendSiteSlices(loginbase):
 371         if isPendingSite(loginbase):
 372                 msg = "INFO: suspendSiteSlices: Pending Site (%s)" % loginbase
 373                 print msg
 374                 logger.info(msg)
 375                 return
 376
 377         api = xmlrpclib.Server(auth.server, verbose=False)
 378         for slice in slices(loginbase):
 379                 logger.info("Suspending slice %s" % slice)
 380                 try:
 381                         if not debug:
 382                             if not isSliceExempt(slice):
 383                                     api.AddSliceTag(auth.auth, slice, "enabled", "0")
 384                 except Exception, exc:
 385                         logger.info("suspendSlices:  %s" % exc)
 386
 387 '''
 388 Freeze all site slices.
 389 '''
 390 def suspendSlices(nodename):
 391         loginbase = siteId(nodename)
 392         suspendSiteSlices(loginbase)
 393
 394
 395 def enableSiteSlices(loginbase):
 396         if isPendingSite(loginbase):
 397                 msg = "INFO: enableSiteSlices: Pending Site (%s)" % loginbase
 398                 print msg
 399                 logger.info(msg)
 400                 return
 401
 402         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 403         for slice in slices(loginbase):
 404                 logger.info("Enabling slices %s" % slice)
 405                 try:
 406                         if not debug:
 407                                 slice_list = api.GetSlices(auth.auth, {'name': slice}, None)
 408                                 if len(slice_list) == 0:
 409                                         return
 410                                 slice_id = slice_list[0]['slice_id']
 411                                 l_attr = api.GetSliceTags(auth.auth, {'slice_id': slice_id}, None)
 412                                 for attr in l_attr:
 413                                         if "enabled" == attr['tagname'] and attr['value'] == "0":
 414                                                 logger.info("Deleted enable=0 attribute from slice %s" % slice)
 415                                                 api.DeleteSliceTag(auth.auth, attr['slice_tag_id'])
 416                 except Exception, exc:
 417                         logger.info("enableSiteSlices: %s" % exc)
 418                         print "exception: %s" % exc
 419
 420 def enableSlices(nodename):
 421         loginbase = siteId(nodename)
 422         enableSiteSlices(loginbase)
 423
 424
 425 #I'm commenting this because this really should be a manual process.
 426 #'''
 427 #Enable suspended site slices.
 428 #'''
 429 #def enableSlices(nodename, slicelist):
 430 #       api = xmlrpclib.Server(auth.server, verbose=False)
 431 #       for slice in  slices(siteId(nodename)):
 432 #               logger.info("Suspending slice %s" % slice)
 433 #               api.SliceTagAdd(auth.auth, slice, "plc_slice_state", {"state" : "suspended"})
 434 #
 435 def enableSiteSliceCreation(loginbase):
 436         if isPendingSite(loginbase):
 437                 msg = "INFO: enableSiteSliceCreation: Pending Site (%s)" % loginbase
 438                 print msg
 439                 logger.info(msg)
 440                 return
 441
 442         api = xmlrpclib.Server(auth.server, verbose=False, allow_none=True)
 443         try:
 444                 logger.info("Enabling slice creation for site %s" % loginbase)
 445                 if not debug:
 446                         site = api.GetSites(auth.auth, loginbase)[0]
 447                         if site['enabled'] == False:
 448                                 logger.info("\tcalling UpdateSite(%s, enabled=True)" % loginbase)
 449                                 if not isSiteExempt(loginbase):
 450                                         api.UpdateSite(auth.auth, loginbase, {'enabled': True})
 451         except Exception, exc:
 452                 print "ERROR: enableSiteSliceCreation:  %s" % exc
 453                 logger.info("ERROR: enableSiteSliceCreation:  %s" % exc)
 454
 455 def enableSliceCreation(nodename):
 456         loginbase = siteId(nodename)
 457         enableSiteSliceCreation(loginbase)
 458
 459 def areSlicesEnabled(site):
 460
 461         try:
 462                 slice_list = api.GetSlices(slices(site))
 463                 if len(slice_list) == 0:
 464                         return None
 465                 for slice in slice_list:
 466                         slice_id = slice['slice_id']
 467                         l_attr = api.GetSliceTags({'slice_id': slice_id})
 468                         for attr in l_attr:
 469                                 if "enabled" == attr['tagname'] and attr['value'] == "0":
 470                                         return False
 471
 472         except Exception, exc:
 473                 pass
 474
 475         return True
 476
 477
 478 def isSiteEnabled(site):
 479     try:
 480         site = api.GetSites(site)[0]
 481         return site['enabled']
 482     except:
 483         pass
 484
 485     return True
 486
 487
 488 def isTagCurrent(tags):
 489     if len(tags) > 0:
 490         for tag in tags:
 491             until = tag['value']
 492             if datetime.strptime(until, "%Y%m%d") > datetime.now():
 493                 # NOTE: the 'exempt_until' time is beyond current time
 494                 return True
 495     return False
 496
 497 def isPersonExempt(email):
 498     tags = api.GetPersonTags({'email' : email, 'tagname' : 'exempt_person_until'})
 499     return isTagCurrent(tags)
 500
 501 def isNodeExempt(hostname):
 502     tags = api.GetNodeTags({'hostname' : hostname, 'tagname' : 'exempt_node_until'})
 503     return isTagCurrent(tags)
 504
 505 def isSliceExempt(slicename):
 506     tags = api.GetSliceTags({'name' : slicename, 'tagname' : 'exempt_slice_until'})
 507     return isTagCurrent(tags)
 508
 509 def isSiteExempt(loginbase):
 510     tags = api.GetSiteTags({'login_base' : loginbase, 'tagname' : 'exempt_site_until'})
 511     return isTagCurrent(tags)
 512
 513 '''
 514 Removes site's ability to create slices. Returns previous max_slices
 515 '''
 516 def removeSiteSliceCreation(loginbase):
 517         #print "removeSiteSliceCreation(%s)" % loginbase
 518
 519         if isPendingSite(loginbase):
 520                 msg = "INFO: removeSiteSliceCreation: Pending Site (%s)" % loginbase
 521                 print msg
 522                 logger.info(msg)
 523                 return
 524
 525         api = xmlrpclib.Server(auth.server, verbose=False)
 526         try:
 527                 logger.info("Removing slice creation for site %s" % loginbase)
 528                 if not debug:
 529                         if not isSiteExempt(loginbase):
 530                             api.UpdateSite(auth.auth, loginbase, {'enabled': False})
 531         except Exception, exc:
 532                 logger.info("removeSiteSliceCreation:  %s" % exc)
 533
 534 '''
 535 Removes ability to create slices. Returns previous max_slices
 536 '''
 537 def removeSliceCreation(nodename):
 538         loginbase = siteId(nodename)
 539         removeSiteSliceCreation(loginbase)
 540
 541
 542 '''
 543 QED
 544 '''
 545 #def enableSliceCreation(nodename, maxslices):
 546 #       api = xmlrpclib.Server(auth.server, verbose=False)
 547 #       anon = {'AuthMethod': "anonymous"}
 548 #       siteid = api.AnonAdmQuerySite (anon, {"node_hostname": nodename})
 549 #       if len(siteid) == 1:
 550 #               logger.info("Enabling slice creation for site %s" % siteId(nodename))
 551 #               try:
 552 #                       if not debug:
 553 #                               api.AdmUpdateSite(auth.auth, siteid[0], {"max_slices" : maxslices})
 554 #               except Exception, exc:
 555 #                       logger.info("API:  %s" % exc)
 556 #       else:
 557 #               logger.debug("Cant find site for %s.  Cannot enable creation." % nodename)
 558
 559 def main():
 560         logger.setLevel(logging.DEBUG)
 561         ch = logging.StreamHandler()
 562         ch.setLevel(logging.DEBUG)
 563         formatter = logging.Formatter('logger - %(message)s')
 564         ch.setFormatter(formatter)
 565         logger.addHandler(ch)
 566         #print getpcu("kupl2.ittc.ku.edu")
 567         #print getpcu("planetlab1.cse.msu.edu")
 568         #print getpcu("alice.cs.princeton.edu")
 569         #print nodesDbg()
 570         #nodeBootState("alice.cs.princeton.edu", "boot")
 571         #freezeSite("alice.cs.princeton.edu")
 572         print removeSliceCreation("alice.cs.princeton.edu")
 573         #enableSliceCreation("alice.cs.princeton.edu", 1024)
 574         #print getSiteNodes("princeton")
 575         #print siteId("alice.cs.princeton.edu")
 576         #print nodePOD("alice.cs.princeton.edu")
 577         #print slices("princeton")
 578
 579 if __name__=="__main__":
 580         main()