reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import urllib
  12 import threading, popen2
  13 import array, struct
  14 import plc
  15 import base64
  16 from subprocess import PIPE, Popen
  17 import ssh.pxssh as pxssh
  18 import ssh.pexpect as pexpect
  19 import socket
  20 import moncommands
  21
  22 # Use our versions of telnetlib and pyssh
  23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  24 import telnetlib
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  26 import pyssh
  27
  28 # Timeouts in seconds
  29 TELNET_TIMEOUT = 45
  30
  31 # Event class ID from pcu events
  32 #NODE_POWER_CONTROL = 3
  33
  34 # Monitor user ID
  35 #MONITOR_USER_ID = 11142
  36
  37 import logging
  38 logger = logging.getLogger("monitor")
  39 verbose = 1
  40 #dryrun = 0;
  41
  42 class ExceptionNoTransport(Exception): pass
  43 class ExceptionNotFound(Exception): pass
  44 class ExceptionPassword(Exception): pass
  45 class ExceptionTimeout(Exception): pass
  46 class ExceptionPrompt(Exception): pass
  47 class ExceptionSequence(Exception): pass
  48 class ExceptionReset(Exception): pass
  49 class ExceptionPort(Exception): pass
  50 class ExceptionUsername(Exception): pass
  51
  52 def telnet_answer(telnet, expected, buffer):
  53         global verbose
  54
  55         output = telnet.read_until(expected, TELNET_TIMEOUT)
  56         #if verbose:
  57         #       logger.debug(output)
  58         if output.find(expected) == -1:
  59                 raise ExceptionNotFound, "'%s' not found" % expected
  60         else:
  61                 telnet.write(buffer + "\r\n")
  62
  63
  64 # PCU has model, host, preferred-port, user, passwd,
  65
  66 # This is an object derived directly form the PLCAPI DB fields
  67 class PCU(object):
  68         def __init__(self, plc_pcu_dict):
  69                 for field in ['username', 'password', 'site_id',
  70                                                 'hostname', 'ip',
  71                                                 'pcu_id', 'model',
  72                                                 'node_ids', 'ports', ]:
  73                         if field in plc_pcu_dict:
  74                                 self.__setattr__(field, plc_pcu_dict[field])
  75                         else:
  76                                 raise Exception("No such field %s in PCU object" % field)
  77
  78 # These are the convenience functions build around the PCU object.
  79 class PCUModel(PCU):
  80         def __init__(self, plc_pcu_dict):
  81                 PCU.__init__(self, plc_pcu_dict)
  82                 self.host = self.pcu_name()
  83
  84         def pcu_name(self):
  85                 if self.hostname is not None and self.hostname is not "":
  86                         return self.hostname
  87                 elif self.ip is not None and self.ip is not "":
  88                         return self.ip
  89                 else:
  90                         return None
  91
  92         def nodeidToPort(self, node_id):
  93                 if node_id in self.node_ids:
  94                         for i in range(0, len(self.node_ids)):
  95                                 if node_id == self.node_ids[i]:
  96                                         return self.ports[i]
  97
  98                 raise Exception("No such Node ID: %d" % node_id)
  99
 100 # This class captures the observed pcu records from FindBadPCUs.py
 101 class PCURecord:
 102         def __init__(self, pcu_record_dict):
 103                 for field in ['nodenames', 'portstatus',
 104                                                 'dnsmatch',
 105                                                 'complete_entry', ]:
 106                         if field in pcu_record_dict:
 107                                 if field == "reboot":
 108                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 109                                 else:
 110                                         self.__setattr__(field, pcu_record_dict[field])
 111                         else:
 112                                 raise Exception("No such field %s in pcu record dict" % field)
 113
 114 class Transport:
 115         TELNET = 1
 116         SSH    = 2
 117         HTTP   = 3
 118         IPAL   = 4
 119
 120         TELNET_TIMEOUT = 60
 121
 122         def __init__(self, type, verbose):
 123                 self.type = type
 124                 self.verbose = verbose
 125                 self.transport = None
 126
 127         def open(self, host, username=None, password=None, prompt="User Name"):
 128                 transport = None
 129
 130                 if self.type == self.TELNET:
 131                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 132                         transport.set_debuglevel(self.verbose)
 133                         if username is not None:
 134                                 self.transport = transport
 135                                 self.ifThenSend(prompt, username, ExceptionUsername)
 136
 137                 elif self.type == self.SSH:
 138                         if username is not None:
 139                                 transport = pyssh.Ssh(username, host)
 140                                 transport.set_debuglevel(self.verbose)
 141                                 transport.open()
 142                                 # TODO: have an ssh set_debuglevel() also...
 143                         else:
 144                                 raise Exception("Username cannot be None for ssh transport.")
 145                 elif self.type == self.HTTP:
 146                         self.url = "http://%s:%d/" % (host,80)
 147                         uri = "%s:%d" % (host,80)
 148
 149                         # create authinfo
 150                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 151                         authinfo.add_password (None, uri, username, password)
 152                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 153
 154                         transport = urllib2.build_opener(authhandler)
 155
 156                 else:
 157                         raise Exception("Unknown transport type: %s" % self.type)
 158
 159                 self.transport = transport
 160                 return True
 161
 162         def close(self):
 163                 if self.type == self.TELNET:
 164                         self.transport.close()
 165                 elif self.type == self.SSH:
 166                         self.transport.close()
 167                 elif self.type == self.HTTP:
 168                         pass
 169                 else:
 170                         raise Exception("Unknown transport type %s" % self.type)
 171                 self.transport = None
 172
 173         def sendHTTP(self, resource, data):
 174                 if self.verbose:
 175                         print "POSTing '%s' to %s" % (data,self.url + resource)
 176
 177                 try:
 178                         f = self.transport.open(self.url + resource ,data)
 179                         r = f.read()
 180                         if self.verbose:
 181                                 print r
 182
 183                 except urllib2.URLError,err:
 184                         logger.info('Could not open http connection', err)
 185                         return "http transport error"
 186
 187                 return 0
 188
 189         def sendPassword(self, password, prompt=None):
 190                 if self.type == self.TELNET:
 191                         if prompt == None:
 192                                 self.ifThenSend("Password", password, ExceptionPassword)
 193                         else:
 194                                 self.ifThenSend(prompt, password, ExceptionPassword)
 195                 elif self.type == self.SSH:
 196                         self.ifThenSend("password:", password, ExceptionPassword)
 197                 elif self.type == self.HTTP:
 198                         pass
 199                 else:
 200                         raise Exception("Unknown transport type: %s" % self.type)
 201
 202         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 203
 204                 if self.transport != None:
 205                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 206                         if output.find(expected) == -1:
 207                                 raise ErrorClass, "'%s' not found" % expected
 208                         else:
 209                                 self.transport.write(buffer + "\r\n")
 210                 else:
 211                         raise ExceptionNoTransport("transport object is type None")
 212
 213         def ifElse(self, expected, ErrorClass):
 214                 try:
 215                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 216                 except:
 217                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 218
 219
 220 class PCUControl(Transport,PCUModel,PCURecord):
 221         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 222                 PCUModel.__init__(self, plc_pcu_record)
 223                 PCURecord.__init__(self, plc_pcu_record)
 224                 type = None
 225                 if self.portstatus:
 226                         if '22' in supported_ports and self.portstatus['22'] == "open":
 227                                 type = Transport.SSH
 228                         elif '23' in supported_ports and self.portstatus['23'] == "open":
 229                                 type = Transport.TELNET
 230                         elif '80' in supported_ports and self.portstatus['80'] == "open":
 231                                 type = Transport.HTTP
 232                         elif '443' in supported_ports and self.portstatus['443'] == "open":
 233                                 type = Transport.HTTP
 234                         elif '5869' in supported_ports and self.portstatus['5869'] == "open":
 235                                 # For DRAC cards. Racadm opens this port.
 236                                 type = Transport.HTTP
 237                         elif '9100' in supported_ports and self.portstatus['9100'] == "open":
 238                                 type = Transport.IPAL
 239                         elif '16992' in supported_ports and self.portstatus['16992'] == "open":
 240                                 type = Transport.HTTP
 241                         else:
 242                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 243                 else:
 244                         raise Exception("No Portstatus: No transport because no open ports")
 245                 Transport.__init__(self, type, verbose)
 246
 247         def run(self, node_port, dryrun):
 248                 """ This function is to be defined by the specific PCU instance.  """
 249                 pass
 250
 251         def reboot(self, node_port, dryrun):
 252                 try:
 253                         return self.run(node_port, dryrun)
 254                 except ExceptionNotFound, err:
 255                         return "error: " + str(err)
 256                 except ExceptionPassword, err:
 257                         return "password exception: " + str(err)
 258                 except ExceptionTimeout, err:
 259                         return "timeout exception: " + str(err)
 260                 except ExceptionUsername, err:
 261                         return "exception: no username prompt: " + str(err)
 262                 except ExceptionSequence, err:
 263                         return "sequence error: " + str(err)
 264                 except ExceptionPrompt, err:
 265                         return "prompt exception: " + str(err)
 266                 except ExceptionPort, err:
 267                         return "no ports exception: " + str(err)
 268                 except socket.error, err:
 269                         return "socket error: timeout: " + str(err)
 270                 except EOFError, err:
 271                         if self.verbose:
 272                                 logger.debug("reboot: EOF")
 273                                 logger.debug(err)
 274                         self.transport.close()
 275                         import traceback
 276                         traceback.print_exc()
 277                         return "EOF connection reset" + str(err)
 278
 279 class IPAL(PCUControl):
 280         """
 281                 This now uses a proprietary format for communicating with the PCU.  I
 282                 prefer it to Telnet, and Web access, since it's much lighter weight
 283                 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
 284         """
 285
 286         def format_msg(self, data, cmd):
 287                 esc = chr(int('1b',16))
 288                 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
 289
 290         def recv_noblock(self, s, count):
 291                 import errno
 292
 293                 try:
 294                         # TODO: make sleep backoff, before stopping.
 295                         time.sleep(4)
 296                         ret = s.recv(count, socket.MSG_DONTWAIT)
 297                 except socket.error, e:
 298                         if e[0] == errno.EAGAIN:
 299                                 raise Exception(e[1])
 300                         else:
 301                                 # TODO: not other exceptions.
 302                                 raise Exception(e)
 303                 return ret
 304
 305         def run(self, node_port, dryrun):
 306                 import errno
 307
 308                 power_on = False
 309
 310                 print "open socket"
 311                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 312                 try:
 313                         print "connect"
 314                         s.connect((self.host, 9100))
 315                 except socket.error, e:
 316                         s.close()
 317                         if e[0] == errno.ECONNREFUSED:
 318                                 # cannot connect to remote host
 319                                 raise Exception(e[1])
 320                         else:
 321                                 # TODO: what other conditions are there?
 322                                 raise Exception(e)
 323
 324                 # get current status
 325                 print "Checking status"
 326                 s.send(self.format_msg("", 'O'))
 327                 ret = self.recv_noblock(s, 8)
 328                 print "Current status is '%s'" % ret
 329
 330                 if ret == '':
 331                         raise Exception("Status returned 'another session already open' %s : %s" % (node_port, ret))
 332
 333
 334                 if node_port < len(ret):
 335                         status = ret[node_port]
 336                         if status == '1':
 337                                 # up
 338                                 power_on = True
 339                         elif status == '0':
 340                                 # down
 341                                 power_on = False
 342                         else:
 343                                 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 344                 else:
 345                         raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 346
 347
 348                 if not dryrun:
 349                         if power_on:
 350                                 print "Pulsing %s" % node_port
 351                                 s.send(self.format_msg("%s" % node_port, 'P'))
 352                         else:
 353                                 # NOTE: turn power on ; do not pulse the port.
 354                                 print "Power was off, so turning on ..."
 355                                 s.send(self.format_msg("%s" % node_port, 'E'))
 356                                 #s.send(self.format_msg("%s" % node_port, 'P'))
 357
 358                         print "Receiving response."
 359                         ret = self.recv_noblock(s, 8)
 360                         print "Current status is '%s'" % ret
 361
 362                         if node_port < len(ret):
 363                                 status = ret[node_port]
 364                                 if status == '1':
 365                                         # up
 366                                         power_on = True
 367                                 elif status == '0':
 368                                         # down
 369                                         power_on = False
 370                                 else:
 371                                         raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 372                         else:
 373                                 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 374
 375                         if power_on:
 376                                 return 0
 377                         else:
 378                                 return "Failed Power On"
 379
 380                 s.close()
 381                 return 0
 382
 383 # TELNET version of protocol...
 384 #               #self.open(self.host)
 385 #               ## XXX Some iPals require you to hit Enter a few times first
 386 #               #self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 387 #               # Login
 388 #               self.ifThenSend("Password >", self.password, ExceptionPassword)
 389 #               self.transport.write("\r\n\r\n")
 390 #               if not dryrun: # P# - Pulse relay
 391 #                       print "node_port %s" % node_port
 392 #                       self.ifThenSend("Enter >",
 393 #                                                       "P7", # % node_port,
 394 #                                                       ExceptionNotFound)
 395 #                       print "send newlines"
 396 #                       self.transport.write("\r\n\r\n")
 397 #                       print "after new lines"
 398 #               # Get the next prompt
 399 #               print "wait for enter"
 400 #               self.ifElse("Enter >", ExceptionTimeout)
 401 #               print "closing "
 402 #               self.close()
 403 #               return 0
 404
 405 class APCEurope(PCUControl):
 406         def run(self, node_port, dryrun):
 407                 self.open(self.host, self.username)
 408                 self.sendPassword(self.password)
 409
 410                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 411                 self.ifThenSend("\r\n> ", "2")
 412                 self.ifThenSend("\r\n> ", str(node_port))
 413                 # 3- Immediate Reboot
 414                 self.ifThenSend("\r\n> ", "3")
 415
 416                 if not dryrun:
 417                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 418                                                         "YES\r\n",
 419                                                         ExceptionSequence)
 420                 else:
 421                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 422                                                         "", ExceptionSequence)
 423                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 424
 425                 self.close()
 426                 return 0
 427
 428 class APCBrazil(PCUControl):
 429         def run(self, node_port, dryrun):
 430                 self.open(self.host, self.username)
 431                 self.sendPassword(self.password)
 432
 433                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 434                 self.ifThenSend("\r\n> ", str(node_port))
 435                 # 4- Immediate Reboot
 436                 self.ifThenSend("\r\n> ", "4")
 437
 438                 if not dryrun:
 439                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 440                                                         "YES\r\n",
 441                                                         ExceptionSequence)
 442                 else:
 443                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 444                                                         "", ExceptionSequence)
 445                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 446
 447                 self.close()
 448                 return 0
 449
 450 class APCBerlin(PCUControl):
 451         def run(self, node_port, dryrun):
 452                 self.open(self.host, self.username)
 453                 self.sendPassword(self.password)
 454
 455                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 456                 self.ifThenSend("\r\n> ", "2")
 457                 self.ifThenSend("\r\n> ", "1")
 458                 self.ifThenSend("\r\n> ", str(node_port))
 459                 # 3- Immediate Reboot
 460                 self.ifThenSend("\r\n> ", "3")
 461
 462                 if not dryrun:
 463                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 464                                                         "YES\r\n",
 465                                                         ExceptionSequence)
 466                 else:
 467                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 468                                                         "", ExceptionSequence)
 469                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 470
 471                 self.close()
 472                 return 0
 473
 474 class APCFolsom(PCUControl):
 475         def run(self, node_port, dryrun):
 476                 self.open(self.host, self.username)
 477                 self.sendPassword(self.password)
 478
 479                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 480                 self.ifThenSend("\r\n> ", "2")
 481                 self.ifThenSend("\r\n> ", "1")
 482                 self.ifThenSend("\r\n> ", str(node_port))
 483                 self.ifThenSend("\r\n> ", "1")
 484
 485                 # 3- Immediate Reboot
 486                 self.ifThenSend("\r\n> ", "3")
 487
 488                 if not dryrun:
 489                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 490                                                         "YES\r\n",
 491                                                         ExceptionSequence)
 492                 else:
 493                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 494                                                         "", ExceptionSequence)
 495                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 496
 497                 self.close()
 498                 return 0
 499
 500 class APCMaster(PCUControl):
 501         def run(self, node_port, dryrun):
 502                 print "Rebooting %s" % self.host
 503                 self.open(self.host, self.username)
 504                 self.sendPassword(self.password)
 505
 506                 # 1- Device Manager
 507                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 508                 # 3- Outlet Control/Config
 509                 self.ifThenSend("\r\n> ", "3")
 510                 # n- Outlet n
 511                 self.ifThenSend("\r\n> ", str(node_port))
 512                 # 1- Control Outlet
 513                 self.ifThenSend("\r\n> ", "1")
 514                 # 3- Immediate Reboot
 515                 self.ifThenSend("\r\n> ", "3")
 516
 517                 if not dryrun:
 518                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 519                                                         "YES\r\n",
 520                                                         ExceptionSequence)
 521                 else:
 522                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 523                                                         "", ExceptionSequence)
 524                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 525
 526                 self.close()
 527                 return 0
 528
 529 class APC(PCUControl):
 530         def __init__(self, plc_pcu_record, verbose):
 531                 PCUControl.__init__(self, plc_pcu_record, verbose)
 532
 533                 self.master = APCMaster(plc_pcu_record, verbose)
 534                 self.folsom = APCFolsom(plc_pcu_record, verbose)
 535                 self.europe = APCEurope(plc_pcu_record, verbose)
 536
 537         def run(self, node_port, dryrun):
 538                 try_again = True
 539                 sleep_time = 1
 540
 541                 for pcu in [self.master, self.europe, self.folsom]:
 542                         if try_again:
 543                                 try:
 544                                         print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
 545                                         try_again = False
 546                                         print "sleeping 5"
 547                                         time.sleep(sleep_time)
 548                                         ret = pcu.reboot(node_port, dryrun)
 549                                 except ExceptionSequence, err:
 550                                         del pcu
 551                                         sleep_time = 130
 552                                         try_again = True
 553
 554                 if try_again:
 555                         return "Unknown reboot sequence for APC PCU"
 556                 else:
 557                         return ret
 558
 559 class IntelAMT(PCUControl):
 560         def run(self, node_port, dryrun):
 561
 562                 cmd = moncommands.CMD()
 563                 #[cmd_str = "IntelAMTSDK/Samples/RemoteControl/remoteControl"
 564                 cmd_str = "cmdamt/remoteControl"
 565
 566                 if dryrun:
 567                         # NOTE: -p checks the power state of the host.
 568                         # TODO: parse the output to find out if it's ok or not.
 569                         cmd_str += " -p http://%s:16992/RemoteControlService  -user admin -pass '%s' " % (self.host, self.password )
 570                 else:
 571                         cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
 572
 573                 print cmd_str
 574                 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
 575
 576 class DRACRacAdm(PCUControl):
 577         def run(self, node_port, dryrun):
 578
 579                 print "trying racadm_reboot..."
 580                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 581
 582                 return 0
 583
 584 class DRAC(PCUControl):
 585         def run(self, node_port, dryrun):
 586                 self.open(self.host, self.username)
 587                 self.sendPassword(self.password)
 588
 589                 print "logging in..."
 590                 self.transport.write("\r\n")
 591                 # Testing Reboot ?
 592                 if dryrun:
 593                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 594                 else:
 595                         # Reset this machine
 596                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 597
 598                 self.ifThenSend("[%s]#" % self.username, "exit")
 599
 600                 self.close()
 601                 return 0
 602
 603 class HPiLO(PCUControl):
 604         def run(self, node_port, dryrun):
 605                 self.open(self.host, self.username)
 606                 self.sendPassword(self.password)
 607
 608                 # </>hpiLO->
 609                 self.ifThenSend("</>hpiLO->", "cd system1")
 610
 611                 # Reboot Outlet  N        (Y/N)?
 612                 if dryrun:
 613                         self.ifThenSend("</system1>hpiLO->", "POWER")
 614                 else:
 615                         # Reset this machine
 616                         self.ifThenSend("</system1>hpiLO->", "reset")
 617
 618                 self.ifThenSend("</system1>hpiLO->", "exit")
 619
 620                 self.close()
 621                 return 0
 622
 623
 624 class HPiLOHttps(PCUControl):
 625         def run(self, node_port, dryrun):
 626
 627                 locfg = moncommands.CMD()
 628                 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 629                                         self.host, "iloxml/Get_Network.xml",
 630                                         self.username, self.password)
 631                 sout, serr = locfg.run_noexcept(cmd)
 632
 633                 if sout.strip() != "":
 634                         print "sout: %s" % sout.strip()
 635                         return sout.strip()
 636
 637                 if not dryrun:
 638                         locfg = moncommands.CMD()
 639                         cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 640                                                 self.host, "iloxml/Reset_Server.xml",
 641                                                 self.username, self.password)
 642                         sout, serr = locfg.run_noexcept(cmd)
 643
 644                         if sout.strip() != "":
 645                                 print "sout: %s" % sout.strip()
 646                                 #return sout.strip()
 647                 return 0
 648
 649 class BayTechAU(PCUControl):
 650         def run(self, node_port, dryrun):
 651                 self.open(self.host, self.username, None, "Enter user name:")
 652                 self.sendPassword(self.password, "Enter Password:")
 653
 654                 #self.ifThenSend("RPC-16>", "Status")
 655                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 656
 657                 # Reboot Outlet  N        (Y/N)?
 658                 if dryrun:
 659                         self.ifThenSend("(Y/N)?", "N")
 660                 else:
 661                         self.ifThenSend("(Y/N)?", "Y")
 662                 self.ifThenSend("RPC3-NC>", "")
 663
 664                 self.close()
 665                 return 0
 666
 667 class BayTechGeorgeTown(PCUControl):
 668         def run(self, node_port, dryrun):
 669                 self.open(self.host, self.username, None, "Enter user name:")
 670                 self.sendPassword(self.password, "Enter Password:")
 671
 672                 #self.ifThenSend("RPC-16>", "Status")
 673
 674                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 675
 676                 # Reboot Outlet  N        (Y/N)?
 677                 if dryrun:
 678                         self.ifThenSend("(Y/N)?", "N")
 679                 else:
 680                         self.ifThenSend("(Y/N)?", "Y")
 681                 self.ifThenSend("RPC-16>", "")
 682
 683                 self.close()
 684                 return 0
 685
 686 class BayTechCtrlCUnibe(PCUControl):
 687         """
 688                 For some reason, these units let you log in fine, but they hang
 689                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 690                 why.
 691         """
 692         def run(self, node_port, dryrun):
 693                 print "BayTechCtrlC %s" % self.host
 694
 695                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 696                 s = pxssh.pxssh()
 697                 if not s.login(self.host, self.username, self.password, ssh_options):
 698                         raise ExceptionPassword("Invalid Password")
 699                 # Otherwise, the login succeeded.
 700
 701                 # Send a ctrl-c to the remote process.
 702                 print "sending ctrl-c"
 703                 s.send(chr(3))
 704
 705                 # Control Outlets  (5 ,1).........5
 706                 try:
 707                         print s
 708                         print "Enter Request" in s.before
 709                         index = s.expect("Enter Request")
 710
 711                         if index == 0:
 712                                 print "3"
 713                                 s.send("3\r\n")
 714                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 715                                 if index == 1:
 716                                         s.send(self.username + "\r\n")
 717                                         index = s.expect(["DS-RPC>"])
 718
 719                                 if index == 0:
 720                                         print "Reboot %d" % node_port
 721                                         s.send("Reboot %d\r\n" % node_port)
 722
 723                                         index = s.expect(["(Y/N)?"])
 724                                         if index == 0:
 725                                                 if dryrun:
 726                                                         print "sending N"
 727                                                         s.send("N\r\n")
 728                                                 else:
 729                                                         print "sending Y"
 730                                                         s.send("Y\r\n")
 731
 732                                 #index = s.expect(["DS-RPC>"])
 733                                 #print "got prompt back"
 734
 735                         s.close()
 736
 737                 except pexpect.EOF:
 738                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 739                 except pexpect.TIMEOUT:
 740                         raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
 741
 742                 return 0
 743
 744 class BayTechCtrlC(PCUControl):
 745         """
 746                 For some reason, these units let you log in fine, but they hang
 747                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 748                 why.
 749         """
 750         def run(self, node_port, dryrun):
 751                 print "BayTechCtrlC %s" % self.host
 752
 753                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 754                 s = pxssh.pxssh()
 755                 if not s.login(self.host, self.username, self.password, ssh_options):
 756                         raise ExceptionPassword("Invalid Password")
 757                 # Otherwise, the login succeeded.
 758
 759                 # Send a ctrl-c to the remote process.
 760                 print "sending ctrl-c"
 761                 s.send(chr(3))
 762
 763                 # Control Outlets  (5 ,1).........5
 764                 try:
 765                         index = s.expect(["Enter Request :"])
 766
 767                         if index == 0:
 768                                 print "5"
 769                                 s.send("5\r\n")
 770                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 771                                 if index == 1:
 772                                         print "sending username"
 773                                         s.send(self.username + "\r\n")
 774                                         index = s.expect(["DS-RPC>"])
 775
 776                                 if index == 0:
 777                                         print "Reboot %d" % node_port
 778                                         s.send("Reboot %d\r\n" % node_port)
 779
 780                                         index = s.expect(["(Y/N)?"])
 781                                         if index == 0:
 782                                                 if dryrun:
 783                                                         print "sending N"
 784                                                         s.send("N\r\n")
 785                                                 else:
 786                                                         print "sending Y"
 787                                                         s.send("Y\r\n")
 788
 789                                 # NOTE: for some reason, the script times out with the
 790                                 # following line.  In manual tests, it works correctly, but
 791                                 # with automated tests, evidently it fails.
 792                                 #index = s.expect(["DS-RPC>"])
 793                                 #print "got prompt back"
 794
 795                         s.close()
 796
 797                 except pexpect.EOF:
 798                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 799                 except pexpect.TIMEOUT:
 800                         raise ExceptionPrompt("Timeout before Prompt")
 801
 802                 return 0
 803
 804 class BayTech(PCUControl):
 805         def run(self, node_port, dryrun):
 806                 self.open(self.host, self.username)
 807                 self.sendPassword(self.password)
 808
 809                 # Control Outlets  (5 ,1).........5
 810                 self.ifThenSend("Enter Request :", "5")
 811
 812                 # Reboot N
 813                 try:
 814                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 815                 except ExceptionNotFound, msg:
 816                         # one machine is configured to ask for a username,
 817                         # even after login...
 818                         print "msg: %s" % msg
 819                         self.transport.write(self.username + "\r\n")
 820                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 821
 822                 # Reboot Outlet  N        (Y/N)?
 823                 if dryrun:
 824                         self.ifThenSend("(Y/N)?", "N")
 825                 else:
 826                         self.ifThenSend("(Y/N)?", "Y")
 827                 self.ifThenSend("DS-RPC>", "")
 828
 829                 self.close()
 830                 return 0
 831
 832 class WTIIPS4(PCUControl):
 833         def run(self, node_port, dryrun):
 834                 self.open(self.host)
 835                 self.sendPassword(self.password, "Enter Password:")
 836
 837                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 838                 if not dryrun:
 839                         self.ifThenSend("Sure? (Y/N): ", "N")
 840                 else:
 841                         self.ifThenSend("Sure? (Y/N): ", "Y")
 842
 843                 self.ifThenSend("IPS> ", "")
 844
 845                 self.close()
 846                 return 0
 847
 848 class ePowerSwitchGood(PCUControl):
 849         # NOTE:
 850         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
 851         #               For some reason this both doesn't work and in some cases, actually
 852         #               hangs the PCU.  Definitely not what we want.
 853         #
 854         #               The code below is much simpler.  Just letting things fail first,
 855         #               and then, trying again with authentication string in the header.
 856         #
 857         def run(self, node_port, dryrun):
 858                 self.transport = None
 859                 self.url = "http://%s:%d/" % (self.host,80)
 860                 uri = "%s:%d" % (self.host,80)
 861
 862                 req = urllib2.Request(self.url)
 863                 try:
 864                         handle = urllib2.urlopen(req)
 865                 except IOError, e:
 866                         # NOTE: this is expected to fail initially
 867                         pass
 868                 else:
 869                         print self.url
 870                         print "-----------"
 871                         print handle.read()
 872                         print "-----------"
 873                         return "ERROR: not protected by HTTP authentication"
 874
 875                 if not hasattr(e, 'code') or e.code != 401:
 876                         return "ERROR: failed for: %s" % str(e)
 877
 878                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
 879                 # NOTE: assuming basic realm authentication.
 880                 authheader = "Basic %s" % base64data
 881                 req.add_header("Authorization", authheader)
 882
 883                 try:
 884                         f = urllib2.urlopen(req)
 885                 except IOError, e:
 886                         # failing here means the User/passwd is wrong (hopefully)
 887                         raise ExceptionPassword("Incorrect username/password")
 888
 889                 # NOTE: after verifying that the user/password is correct,
 890                 #               actually reboot the given node.
 891                 if not dryrun:
 892                         try:
 893                                 data = urllib.urlencode({'P%d' % node_port : "r"})
 894                                 req = urllib2.Request(self.url + "cmd.html")
 895                                 req.add_header("Authorization", authheader)
 896                                 # add data to handler,
 897                                 f = urllib2.urlopen(req, data)
 898                                 if self.verbose: print f.read()
 899                         except:
 900                                 import traceback; traceback.print_exc()
 901
 902                                 # fetch url one more time on cmd.html, econtrol.html or whatever.
 903                                 # pass
 904                 else:
 905                         if self.verbose: print f.read()
 906
 907                 self.close()
 908                 return 0
 909
 910 class CustomPCU(PCUControl):
 911         def run(self, node_port, dryrun):
 912                 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
 913
 914                 if not dryrun:
 915                         # Turn host off, then on
 916                         formstr = "plab%s=off" % node_port
 917                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 918                         time.sleep(5)
 919                         formstr = "plab%s=on" % node_port
 920                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 921                 else:
 922                         os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
 923
 924
 925 class ePowerSwitchOld(PCUControl):
 926         def run(self, node_port, dryrun):
 927                 self.url = "http://%s:%d/" % (self.host,80)
 928                 uri = "%s:%d" % (self.host,80)
 929
 930                 # create authinfo
 931                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 932                 authinfo.add_password (None, uri, self.username, self.password)
 933                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 934
 935                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 936                 transport = urllib2.build_opener(authinfo)
 937                 f = transport.open(self.url)
 938                 if self.verbose: print f.read()
 939
 940                 if not dryrun:
 941                         transport = urllib2.build_opener(authhandler)
 942                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
 943                         if self.verbose: print f.read()
 944
 945                 self.close()
 946                 return 0
 947
 948 class ePowerSwitch(PCUControl):
 949         def run(self, node_port, dryrun):
 950                 self.url = "http://%s:%d/" % (self.host,80)
 951                 uri = "%s:%d" % (self.host,80)
 952
 953                 # TODO: I'm still not sure what the deal is here.
 954                 #               two independent calls appear to need to be made before the
 955                 #               reboot will succeed.  It doesn't seem to be possible to do
 956                 #               this with a single call.  I have no idea why.
 957
 958                 # create authinfo
 959                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 960                 authinfo.add_password (None, uri, self.username, self.password)
 961                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 962
 963                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 964                 transport = urllib2.build_opener()
 965                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
 966                 if self.verbose: print f.read()
 967
 968                 if not dryrun:
 969                         transport = urllib2.build_opener(authhandler)
 970                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
 971                         if self.verbose: print f.read()
 972
 973                 #       data= "P%d=r" % node_port
 974                 #self.open(self.host, self.username, self.password)
 975                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
 976                 #self.sendHTTP("econtrol.html", data)
 977                 #self.sendHTTP("cmd.html", data)
 978
 979                 self.close()
 980                 return 0
 981
 982
 983 ### rebooting european BlackBox PSE boxes
 984 # Thierry Parmentelat - May 11 2005
 985 # tested on 4-ports models known as PSE505-FR
 986 # uses http to POST a data 'P<port>=r'
 987 # relies on basic authentication within http1.0
 988 # first curl-based script was
 989 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
 990 #       http://<hostname>:<http_port>/cmd.html && echo OK
 991
 992 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
 993
 994         global verbose
 995
 996         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
 997         data= "P%d=r" % port_in_pcu
 998         if verbose:
 999                 logger.debug("POSTing '%s' on %s" % (data,url))
1000
1001         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1002         uri = "%s:%d" % (pcu_ip,http_port)
1003         authinfo.add_password (None, uri, username, password)
1004         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1005
1006         opener = urllib2.build_opener(authhandler)
1007         urllib2.install_opener(opener)
1008
1009         if (dryrun):
1010                 return 0
1011
1012         try:
1013                 f = urllib2.urlopen(url,data)
1014
1015                 r= f.read()
1016                 if verbose:
1017                         logger.debug(r)
1018                 return 0
1019
1020         except urllib2.URLError,err:
1021                 logger.info('Could not open http connection', err)
1022                 return "bbpse error"
1023
1024 ### rebooting x10toggle based systems addressed by port
1025 # Marc E. Fiuczynski - May 31 2005
1026 # tested on 4-ports models known as PSE505-FR
1027 # uses ssh and password to login to an account
1028 # that will cause the system to be powercycled.
1029
1030 def x10toggle_reboot(ip, username, password, port, dryrun):
1031         global verbose
1032
1033         ssh = None
1034         try:
1035                 ssh = pyssh.Ssh(username, ip)
1036                 ssh.open()
1037
1038                 # Login
1039                 telnet_answer(ssh, "password:", password)
1040
1041                 if not dryrun:
1042                         # Reboot
1043                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
1044
1045                 # Close
1046                 output = ssh.close()
1047                 if verbose:
1048                         logger.debug(output)
1049                 return 0
1050
1051         except Exception, err:
1052                 if verbose:
1053                         logger.debug(err)
1054                 if ssh:
1055                         output = ssh.close()
1056                         if verbose:
1057                                 logger.debug(output)
1058                 return errno.ETIMEDOUT
1059
1060 ### rebooting Dell systems via RAC card
1061 # Marc E. Fiuczynski - June 01 2005
1062 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1063 #
1064
1065 def runcmd(command, args, username, password, timeout = None):
1066
1067         result = [None]
1068         result_ready = threading.Condition()
1069
1070         def set_result(x):
1071
1072                 result_ready.acquire()
1073                 try:
1074                         result[0] = x
1075                 finally:
1076                         result_ready.notify()
1077                         result_ready.release()
1078
1079         def do_command(command, username, password):
1080
1081                 try:
1082                         # Popen4 is a popen-type class that combines stdout and stderr
1083                         p = popen2.Popen4(command)
1084
1085                         # read all output data
1086                         p.tochild.write("%s\n" % username)
1087                         p.tochild.write("%s\n" % password)
1088                         p.tochild.close()
1089                         data = p.fromchild.read()
1090
1091                         while True:
1092                                 # might get interrupted by a signal in poll() or waitpid()
1093                                 try:
1094                                         retval = p.wait()
1095                                         set_result((retval, data))
1096                                         break
1097                                 except OSError, ex:
1098                                         if ex.errno == errno.EINTR:
1099                                                 continue
1100                                         raise ex
1101                 except Exception, ex:
1102                         set_result(ex)
1103
1104         if args:
1105                 command = " ".join([command] + args)
1106
1107         worker = threading.Thread(target = do_command, args = (command, username, password, ))
1108         worker.setDaemon(True)
1109         result_ready.acquire()
1110         worker.start()
1111         result_ready.wait(timeout)
1112         try:
1113                 if result == [None]:
1114                         raise Exception, "command timed-out: '%s'" % command
1115         finally:
1116                 result_ready.release()
1117         result = result[0]
1118
1119         if isinstance(result, Exception):
1120                 raise result
1121         else:
1122                 (retval, data) = result
1123                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1124                         return data
1125                 else:
1126                         out = "system command ('%s') " % command
1127                         if os.WIFEXITED(retval):
1128                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1129                         else:
1130                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1131                         if data:
1132                                 out += "; output follows:\n" + data
1133                         raise Exception, out
1134
1135 def racadm_reboot(host, username, password, port, dryrun):
1136         global verbose
1137
1138         ip = socket.gethostbyname(host)
1139         try:
1140                 cmd = "/usr/sbin/racadm"
1141                 os.stat(cmd)
1142                 if not dryrun:
1143                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1144                                 username, password)
1145                 else:
1146                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1147                                 username, password)
1148
1149                 print "RUNCMD: %s" % output
1150                 if verbose:
1151                         logger.debug(output)
1152                 return 0
1153
1154         except Exception, err:
1155                 logger.debug("runcmd raised exception %s" % err)
1156                 if verbose:
1157                         logger.debug(err)
1158                 return -1
1159
1160 def pcu_name(pcu):
1161         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1162                 return pcu['hostname']
1163         elif pcu['ip'] is not None and pcu['ip'] is not "":
1164                 return pcu['ip']
1165         else:
1166                 return None
1167
1168 #import database
1169 from monitor import database
1170 fb = None
1171
1172 def get_pcu_values(pcu_id):
1173         global fb
1174         if fb == None:
1175                 # this shouldn't be loaded each time...
1176                 fb = database.dbLoad("findbadpcus")
1177
1178         try:
1179                 values = fb['nodes']["id_%s" % pcu_id]['values']
1180         except:
1181                 values = None
1182
1183         return values
1184
1185 def reboot(nodename):
1186         return reboot_policy(nodename, True, False)
1187
1188 def reboot_policy(nodename, continue_probe, dryrun):
1189         global verbose
1190
1191         pcu = plc.getpcu(nodename)
1192         if not pcu:
1193                 logger.debug("no pcu for %s" % hostname)
1194                 print "no pcu for %s" % hostname
1195                 return False # "%s has no pcu" % nodename
1196
1197         values = get_pcu_values(pcu['pcu_id'])
1198         if values == None:
1199                 logger.debug("No values for pcu probe %s" % hostname)
1200                 print "No values for pcu probe %s" % hostname
1201                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1202
1203         # Try the PCU first
1204         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1205
1206         ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1207
1208         if ret != 0:
1209                 print ret
1210                 return False
1211         else:
1212                 print "return true"
1213                 return True
1214
1215 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1216         rb_ret = ""
1217
1218         try:
1219                 # DataProbe iPal (many sites)
1220                 if  continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1221                         ipal = IPAL(values, verbose, ['23', '80', '9100'])
1222                         rb_ret = ipal.reboot(values[nodename], dryrun)
1223
1224                 # APC Masterswitch (Berkeley)
1225                 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1226                                                                   values['model'].find("Masterswitch") >= 0 ):
1227                         print values
1228
1229                         # TODO: make a more robust version of APC
1230                         if values['pcu_id'] in [1163,1055,1111,1231,1113,1127,1128,1148]:
1231                                 apc = APCEurope(values, verbose, ['22', '23'])
1232                                 rb_ret = apc.reboot(values[nodename], dryrun)
1233
1234                         elif values['pcu_id'] in [1110,86]:
1235                                 apc = APCBrazil(values, verbose, ['22', '23'])
1236                                 rb_ret = apc.reboot(values[nodename], dryrun)
1237
1238                         elif values['pcu_id'] in [1221,1225]:
1239                                 apc = APCBerlin(values, verbose, ['22', '23'])
1240                                 rb_ret = apc.reboot(values[nodename], dryrun)
1241
1242                         elif values['pcu_id'] in [1173,1221,1220]:
1243                                 apc = APCFolsom(values, verbose, ['22', '23'])
1244                                 rb_ret = apc.reboot(values[nodename], dryrun)
1245
1246                         else:
1247                                 apc = APCMaster(values, verbose, ['22', '23'])
1248                                 rb_ret = apc.reboot(values[nodename], dryrun)
1249
1250                 # BayTech DS4-RPC
1251                 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1252                         if values['pcu_id'] in [1237,1052,1209,1002,1008,1041,1013,1022]:
1253                                 # These  require a 'ctrl-c' to be sent...
1254                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1255                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1256
1257                         elif values['pcu_id'] in [93]:
1258                                 baytech = BayTechAU(values, verbose, ['22', '23'])
1259                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1260
1261                         elif values['pcu_id'] in [1057]:
1262                                 # These  require a 'ctrl-c' to be sent...
1263                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1264                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1265
1266                         elif values['pcu_id'] in [1012]:
1267                                 # This pcu sometimes doesn't present the 'Username' prompt,
1268                                 # unless you immediately try again...
1269                                 try:
1270                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1271                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1272                                 except:
1273                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1274                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1275                         else:
1276                                 baytech = BayTech(values, verbose, ['22', '23'])
1277                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1278
1279                 # iLO
1280                 elif continue_probe and values['model'].find("ilo") >= 0:
1281                         try:
1282                                 hpilo = HPiLO(values, verbose, ['22'])
1283                                 rb_ret = hpilo.reboot(0, dryrun)
1284                                 if rb_ret != 0:
1285                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1286                                         rb_ret = hpilo.reboot(0, dryrun)
1287                         except:
1288                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1289                                 rb_ret = hpilo.reboot(0, dryrun)
1290
1291                 # DRAC ssh
1292                 elif continue_probe and values['model'].find("DRAC") >= 0:
1293                         # TODO: I don't think DRACRacAdm will throw an exception for the
1294                         # default method to catch...
1295                         try:
1296                                 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1297                                 rb_ret = drac.reboot(0, dryrun)
1298                         except:
1299                                 drac = DRAC(values, verbose, ['22'])
1300                                 rb_ret = drac.reboot(0, dryrun)
1301
1302                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1303                                 wti = WTIIPS4(values, verbose, ['23'])
1304                                 rb_ret = wti.reboot(values[nodename], dryrun)
1305
1306                 elif continue_probe and values['model'].find("AMT") >= 0:
1307                                 amt = IntelAMT(values, verbose, ['16992'])
1308                                 rb_ret = amt.reboot(values[nodename], dryrun)
1309
1310                 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1311                 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1312                         # TODO: allow a different port than http 80.
1313                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1314                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1315                         elif values['pcu_id'] in [1003]:
1316                                 # OLD EPOWER
1317                                 print "OLD EPOWER"
1318                                 eps = ePowerSwitch(values, verbose, ['80'])
1319                         else:
1320                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1321
1322                         rb_ret = eps.reboot(values[nodename], dryrun)
1323                 elif continue_probe and values['pcu_id'] in [1122]:
1324                         custom = CustomPCU(values, verbose, ['80', '443'])
1325                         custom.reboot(values[nodename], dryrun)
1326
1327                 elif continue_probe:
1328                         rb_ret = "Unsupported_PCU"
1329
1330                 elif continue_probe == False:
1331                         if 'portstatus' in values:
1332                                 rb_ret = "NetDown"
1333                         else:
1334                                 rb_ret = "Not_Run"
1335                 else:
1336                         rb_ret = -1
1337
1338         except ExceptionPort, err:
1339                 rb_ret = str(err)
1340
1341         return rb_ret
1342         # ????
1343         #elif continue_probe and values['protocol'] == "racadm" and \
1344         #               values['model'] == "RAC":
1345         #       rb_ret = racadm_reboot(pcu_name(values),
1346         #                                                                 values['username'],
1347         #                                                                 values['password'],
1348         #                                                                 pcu[nodename],
1349         #                                                                 dryrun)
1350
1351 def main():
1352         logger.setLevel(logging.DEBUG)
1353         ch = logging.StreamHandler()
1354         ch.setLevel(logging.DEBUG)
1355         formatter = logging.Formatter('LOGGER - %(message)s')
1356         ch.setFormatter(formatter)
1357         logger.addHandler(ch)
1358
1359         try:
1360                 if "test" in sys.argv:
1361                         dryrun = True
1362                 else:
1363                         dryrun = False
1364
1365                 for node in sys.argv[1:]:
1366                         if node == "test": continue
1367
1368                         print "Rebooting %s" % node
1369                         if reboot_policy(node, True, dryrun):
1370                                 print "success"
1371                         else:
1372                                 print "failed"
1373         except Exception, err:
1374                 import traceback; traceback.print_exc()
1375                 print err
1376
1377 if __name__ == '__main__':
1378         import plc
1379         logger = logging.getLogger("monitor")
1380         main()