reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import urllib
  12 import threading, popen2
  13 import array, struct
  14 #from socket import *
  15 import socket
  16 import plc
  17 import base64
  18 from subprocess import PIPE, Popen
  19 import ssh.pxssh as pxssh
  20 import ssh.pexpect as pexpect
  21 import socket
  22 import moncommands
  23
  24 # Use our versions of telnetlib and pyssh
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  26 import telnetlib
  27 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  28 import pyssh
  29
  30 # Timeouts in seconds
  31 TELNET_TIMEOUT = 45
  32
  33 # Event class ID from pcu events
  34 #NODE_POWER_CONTROL = 3
  35
  36 # Monitor user ID
  37 #MONITOR_USER_ID = 11142
  38
  39 import logging
  40 logger = logging.getLogger("monitor")
  41 verbose = 1
  42 #dryrun = 0;
  43
  44 class ExceptionNoTransport(Exception): pass
  45 class ExceptionNotFound(Exception): pass
  46 class ExceptionPassword(Exception): pass
  47 class ExceptionTimeout(Exception): pass
  48 class ExceptionPrompt(Exception): pass
  49 class ExceptionSequence(Exception): pass
  50 class ExceptionReset(Exception): pass
  51 class ExceptionPort(Exception): pass
  52 class ExceptionUsername(Exception): pass
  53
  54 def telnet_answer(telnet, expected, buffer):
  55         global verbose
  56
  57         output = telnet.read_until(expected, TELNET_TIMEOUT)
  58         #if verbose:
  59         #       logger.debug(output)
  60         if output.find(expected) == -1:
  61                 raise ExceptionNotFound, "'%s' not found" % expected
  62         else:
  63                 telnet.write(buffer + "\r\n")
  64
  65
  66 # PCU has model, host, preferred-port, user, passwd,
  67
  68 # This is an object derived directly form the PLCAPI DB fields
  69 class PCU(object):
  70         def __init__(self, plc_pcu_dict):
  71                 for field in ['username', 'password', 'site_id',
  72                                                 'hostname', 'ip',
  73                                                 'pcu_id', 'model',
  74                                                 'node_ids', 'ports', ]:
  75                         if field in plc_pcu_dict:
  76                                 self.__setattr__(field, plc_pcu_dict[field])
  77                         else:
  78                                 raise Exception("No such field %s in PCU object" % field)
  79
  80 # These are the convenience functions build around the PCU object.
  81 class PCUModel(PCU):
  82         def __init__(self, plc_pcu_dict):
  83                 PCU.__init__(self, plc_pcu_dict)
  84                 self.host = self.pcu_name()
  85
  86         def pcu_name(self):
  87                 if self.hostname is not None and self.hostname is not "":
  88                         return self.hostname
  89                 elif self.ip is not None and self.ip is not "":
  90                         return self.ip
  91                 else:
  92                         return None
  93
  94         def nodeidToPort(self, node_id):
  95                 if node_id in self.node_ids:
  96                         for i in range(0, len(self.node_ids)):
  97                                 if node_id == self.node_ids[i]:
  98                                         return self.ports[i]
  99
 100                 raise Exception("No such Node ID: %d" % node_id)
 101
 102 # This class captures the observed pcu records from FindBadPCUs.py
 103 class PCURecord:
 104         def __init__(self, pcu_record_dict):
 105                 for field in ['nodenames', 'portstatus',
 106                                                 'dnsmatch',
 107                                                 'complete_entry', ]:
 108                         if field in pcu_record_dict:
 109                                 if field == "reboot":
 110                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 111                                 else:
 112                                         self.__setattr__(field, pcu_record_dict[field])
 113                         else:
 114                                 raise Exception("No such field %s in pcu record dict" % field)
 115
 116 class Transport:
 117         TELNET = 1
 118         SSH    = 2
 119         HTTP   = 3
 120         IPAL   = 4
 121
 122         TELNET_TIMEOUT = 60
 123
 124         def __init__(self, type, verbose):
 125                 self.type = type
 126                 self.verbose = verbose
 127                 self.transport = None
 128
 129         def open(self, host, username=None, password=None, prompt="User Name"):
 130                 transport = None
 131
 132                 if self.type == self.TELNET:
 133                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 134                         transport.set_debuglevel(self.verbose)
 135                         if username is not None:
 136                                 self.transport = transport
 137                                 self.ifThenSend(prompt, username, ExceptionUsername)
 138
 139                 elif self.type == self.SSH:
 140                         if username is not None:
 141                                 transport = pyssh.Ssh(username, host)
 142                                 transport.set_debuglevel(self.verbose)
 143                                 transport.open()
 144                                 # TODO: have an ssh set_debuglevel() also...
 145                         else:
 146                                 raise Exception("Username cannot be None for ssh transport.")
 147                 elif self.type == self.HTTP:
 148                         self.url = "http://%s:%d/" % (host,80)
 149                         uri = "%s:%d" % (host,80)
 150
 151                         # create authinfo
 152                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 153                         authinfo.add_password (None, uri, username, password)
 154                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 155
 156                         transport = urllib2.build_opener(authhandler)
 157
 158                 else:
 159                         raise Exception("Unknown transport type: %s" % self.type)
 160
 161                 self.transport = transport
 162                 return True
 163
 164         def close(self):
 165                 if self.type == self.TELNET:
 166                         self.transport.close()
 167                 elif self.type == self.SSH:
 168                         self.transport.close()
 169                 elif self.type == self.HTTP:
 170                         pass
 171                 else:
 172                         raise Exception("Unknown transport type %s" % self.type)
 173                 self.transport = None
 174
 175         def sendHTTP(self, resource, data):
 176                 if self.verbose:
 177                         print "POSTing '%s' to %s" % (data,self.url + resource)
 178
 179                 try:
 180                         f = self.transport.open(self.url + resource ,data)
 181                         r = f.read()
 182                         if self.verbose:
 183                                 print r
 184
 185                 except urllib2.URLError,err:
 186                         logger.info('Could not open http connection', err)
 187                         return "http transport error"
 188
 189                 return 0
 190
 191         def sendPassword(self, password, prompt=None):
 192                 if self.type == self.TELNET:
 193                         if prompt == None:
 194                                 self.ifThenSend("Password", password, ExceptionPassword)
 195                         else:
 196                                 self.ifThenSend(prompt, password, ExceptionPassword)
 197                 elif self.type == self.SSH:
 198                         self.ifThenSend("password:", password, ExceptionPassword)
 199                 elif self.type == self.HTTP:
 200                         pass
 201                 else:
 202                         raise Exception("Unknown transport type: %s" % self.type)
 203
 204         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 205
 206                 if self.transport != None:
 207                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 208                         if output.find(expected) == -1:
 209                                 raise ErrorClass, "'%s' not found" % expected
 210                         else:
 211                                 self.transport.write(buffer + "\r\n")
 212                 else:
 213                         raise ExceptionNoTransport("transport object is type None")
 214
 215         def ifElse(self, expected, ErrorClass):
 216                 try:
 217                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 218                 except:
 219                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 220
 221
 222 class PCUControl(Transport,PCUModel,PCURecord):
 223         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 224                 PCUModel.__init__(self, plc_pcu_record)
 225                 PCURecord.__init__(self, plc_pcu_record)
 226                 type = None
 227                 if self.portstatus:
 228                         if '22' in supported_ports and self.portstatus['22'] == "open":
 229                                 type = Transport.SSH
 230                         elif '23' in supported_ports and self.portstatus['23'] == "open":
 231                                 type = Transport.TELNET
 232                         elif '80' in supported_ports and self.portstatus['80'] == "open":
 233                                 type = Transport.HTTP
 234                         elif '443' in supported_ports and self.portstatus['443'] == "open":
 235                                 type = Transport.HTTP
 236                         elif '5869' in supported_ports and self.portstatus['5869'] == "open":
 237                                 # For DRAC cards. Racadm opens this port.
 238                                 type = Transport.HTTP
 239                         elif '9100' in supported_ports and self.portstatus['9100'] == "open":
 240                                 type = Transport.IPAL
 241                         elif '16992' in supported_ports and self.portstatus['16992'] == "open":
 242                                 type = Transport.HTTP
 243                         else:
 244                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 245                 else:
 246                         raise Exception("No Portstatus: No transport because no open ports")
 247                 Transport.__init__(self, type, verbose)
 248
 249         def run(self, node_port, dryrun):
 250                 """ This function is to be defined by the specific PCU instance.  """
 251                 pass
 252
 253         def reboot(self, node_port, dryrun):
 254                 try:
 255                         return self.run(node_port, dryrun)
 256                 except ExceptionNotFound, err:
 257                         return "error: " + str(err)
 258                 except ExceptionPassword, err:
 259                         return "password exception: " + str(err)
 260                 except ExceptionTimeout, err:
 261                         return "timeout exception: " + str(err)
 262                 except ExceptionUsername, err:
 263                         return "exception: no username prompt: " + str(err)
 264                 except ExceptionSequence, err:
 265                         return "sequence error: " + str(err)
 266                 except ExceptionPrompt, err:
 267                         return "prompt exception: " + str(err)
 268                 except ExceptionPort, err:
 269                         return "no ports exception: " + str(err)
 270                 except socket.error, err:
 271                         return "socket error: timeout: " + str(err)
 272                 except EOFError, err:
 273                         if self.verbose:
 274                                 logger.debug("reboot: EOF")
 275                                 logger.debug(err)
 276                         self.transport.close()
 277                         import traceback
 278                         traceback.print_exc()
 279                         return "EOF connection reset" + str(err)
 280
 281 class IPAL(PCUControl):
 282         """
 283                 This now uses a proprietary format for communicating with the PCU.  I
 284                 prefer it to Telnet, and Web access, since it's much lighter weight
 285                 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
 286         """
 287
 288         def format_msg(self, data, cmd):
 289                 esc = chr(int('1b',16))
 290                 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
 291
 292         def recv_noblock(self, s, count):
 293                 import errno
 294
 295                 try:
 296                         # TODO: make sleep backoff, before stopping.
 297                         time.sleep(4)
 298                         ret = s.recv(count, socket.MSG_DONTWAIT)
 299                 except socket.error, e:
 300                         if e[0] == errno.EAGAIN:
 301                                 raise Exception(e[1])
 302                         else:
 303                                 # TODO: not other exceptions.
 304                                 raise Exception(e)
 305                 return ret
 306
 307         def run(self, node_port, dryrun):
 308                 import errno
 309
 310                 power_on = False
 311
 312                 print "open socket"
 313                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 314                 try:
 315                         print "connect"
 316                         s.connect((self.host, 9100))
 317                 except socket.error, e:
 318                         s.close()
 319                         if e[0] == errno.ECONNREFUSED:
 320                                 # cannot connect to remote host
 321                                 raise Exception(e[1])
 322                         else:
 323                                 # TODO: what other conditions are there?
 324                                 raise Exception(e)
 325
 326                 # get current status
 327                 print "Checking status"
 328                 s.send(self.format_msg("", 'O'))
 329                 ret = self.recv_noblock(s, 8)
 330                 print "Current status is '%s'" % ret
 331
 332                 if ret == '':
 333                         raise Exception("Status returned 'another session already open' %s : %s" % (node_port, ret))
 334
 335
 336                 if node_port < len(ret):
 337                         status = ret[node_port]
 338                         if status == '1':
 339                                 # up
 340                                 power_on = True
 341                         elif status == '0':
 342                                 # down
 343                                 power_on = False
 344                         else:
 345                                 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 346                 else:
 347                         raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 348
 349
 350                 if not dryrun:
 351                         if power_on:
 352                                 print "Pulsing %s" % node_port
 353                                 s.send(self.format_msg("%s" % node_port, 'P'))
 354                         else:
 355                                 # NOTE: turn power on ; do not pulse the port.
 356                                 print "Power was off, so turning on ..."
 357                                 s.send(self.format_msg("%s" % node_port, 'E'))
 358                                 #s.send(self.format_msg("%s" % node_port, 'P'))
 359
 360                         print "Receiving response."
 361                         ret = self.recv_noblock(s, 8)
 362                         print "Current status is '%s'" % ret
 363
 364                         if node_port < len(ret):
 365                                 status = ret[node_port]
 366                                 if status == '1':
 367                                         # up
 368                                         power_on = True
 369                                 elif status == '0':
 370                                         # down
 371                                         power_on = False
 372                                 else:
 373                                         raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 374                         else:
 375                                 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 376
 377                         if power_on:
 378                                 return 0
 379                         else:
 380                                 return "Failed Power On"
 381
 382                 s.close()
 383                 return 0
 384
 385 # TELNET version of protocol...
 386 #               #self.open(self.host)
 387 #               ## XXX Some iPals require you to hit Enter a few times first
 388 #               #self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 389 #               # Login
 390 #               self.ifThenSend("Password >", self.password, ExceptionPassword)
 391 #               self.transport.write("\r\n\r\n")
 392 #               if not dryrun: # P# - Pulse relay
 393 #                       print "node_port %s" % node_port
 394 #                       self.ifThenSend("Enter >",
 395 #                                                       "P7", # % node_port,
 396 #                                                       ExceptionNotFound)
 397 #                       print "send newlines"
 398 #                       self.transport.write("\r\n\r\n")
 399 #                       print "after new lines"
 400 #               # Get the next prompt
 401 #               print "wait for enter"
 402 #               self.ifElse("Enter >", ExceptionTimeout)
 403 #               print "closing "
 404 #               self.close()
 405 #               return 0
 406
 407 class APCEurope(PCUControl):
 408         def run(self, node_port, dryrun):
 409                 self.open(self.host, self.username)
 410                 self.sendPassword(self.password)
 411
 412                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 413                 self.ifThenSend("\r\n> ", "2")
 414                 self.ifThenSend("\r\n> ", str(node_port))
 415                 # 3- Immediate Reboot
 416                 self.ifThenSend("\r\n> ", "3")
 417
 418                 if not dryrun:
 419                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 420                                                         "YES\r\n",
 421                                                         ExceptionSequence)
 422                 else:
 423                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 424                                                         "", ExceptionSequence)
 425                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 426
 427                 self.close()
 428                 return 0
 429
 430 class APCBrazil(PCUControl):
 431         def run(self, node_port, dryrun):
 432                 self.open(self.host, self.username)
 433                 self.sendPassword(self.password)
 434
 435                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 436                 self.ifThenSend("\r\n> ", str(node_port))
 437                 # 4- Immediate Reboot
 438                 self.ifThenSend("\r\n> ", "4")
 439
 440                 if not dryrun:
 441                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 442                                                         "YES\r\n",
 443                                                         ExceptionSequence)
 444                 else:
 445                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 446                                                         "", ExceptionSequence)
 447                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 448
 449                 self.close()
 450                 return 0
 451
 452 class APCBerlin(PCUControl):
 453         def run(self, node_port, dryrun):
 454                 self.open(self.host, self.username)
 455                 self.sendPassword(self.password)
 456
 457                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 458                 self.ifThenSend("\r\n> ", "2")
 459                 self.ifThenSend("\r\n> ", "1")
 460                 self.ifThenSend("\r\n> ", str(node_port))
 461                 # 3- Immediate Reboot
 462                 self.ifThenSend("\r\n> ", "3")
 463
 464                 if not dryrun:
 465                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 466                                                         "YES\r\n",
 467                                                         ExceptionSequence)
 468                 else:
 469                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 470                                                         "", ExceptionSequence)
 471                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 472
 473                 self.close()
 474                 return 0
 475
 476 class APCFolsom(PCUControl):
 477         def run(self, node_port, dryrun):
 478                 self.open(self.host, self.username)
 479                 self.sendPassword(self.password)
 480
 481                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 482                 self.ifThenSend("\r\n> ", "2")
 483                 self.ifThenSend("\r\n> ", "1")
 484                 self.ifThenSend("\r\n> ", str(node_port))
 485                 self.ifThenSend("\r\n> ", "1")
 486
 487                 # 3- Immediate Reboot
 488                 self.ifThenSend("\r\n> ", "3")
 489
 490                 if not dryrun:
 491                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 492                                                         "YES\r\n",
 493                                                         ExceptionSequence)
 494                 else:
 495                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 496                                                         "", ExceptionSequence)
 497                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 498
 499                 self.close()
 500                 return 0
 501
 502 class APCMaster(PCUControl):
 503         def run(self, node_port, dryrun):
 504                 print "Rebooting %s" % self.host
 505                 self.open(self.host, self.username)
 506                 self.sendPassword(self.password)
 507
 508                 # 1- Device Manager
 509                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 510                 # 3- Outlet Control/Config
 511                 self.ifThenSend("\r\n> ", "3")
 512                 # n- Outlet n
 513                 self.ifThenSend("\r\n> ", str(node_port))
 514                 # 1- Control Outlet
 515                 self.ifThenSend("\r\n> ", "1")
 516                 # 3- Immediate Reboot
 517                 self.ifThenSend("\r\n> ", "3")
 518
 519                 if not dryrun:
 520                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 521                                                         "YES\r\n",
 522                                                         ExceptionSequence)
 523                 else:
 524                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 525                                                         "", ExceptionSequence)
 526                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 527
 528                 self.close()
 529                 return 0
 530
 531 class APC(PCUControl):
 532         def __init__(self, plc_pcu_record, verbose):
 533                 PCUControl.__init__(self, plc_pcu_record, verbose)
 534
 535                 self.master = APCMaster(plc_pcu_record, verbose)
 536                 self.folsom = APCFolsom(plc_pcu_record, verbose)
 537                 self.europe = APCEurope(plc_pcu_record, verbose)
 538
 539         def run(self, node_port, dryrun):
 540                 try_again = True
 541                 sleep_time = 1
 542
 543                 for pcu in [self.master, self.europe, self.folsom]:
 544                         if try_again:
 545                                 try:
 546                                         print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
 547                                         try_again = False
 548                                         print "sleeping 5"
 549                                         time.sleep(sleep_time)
 550                                         ret = pcu.reboot(node_port, dryrun)
 551                                 except ExceptionSequence, err:
 552                                         del pcu
 553                                         sleep_time = 130
 554                                         try_again = True
 555
 556                 if try_again:
 557                         return "Unknown reboot sequence for APC PCU"
 558                 else:
 559                         return ret
 560
 561 class IntelAMT(PCUControl):
 562         def run(self, node_port, dryrun):
 563
 564                 cmd = moncommands.CMD()
 565                 #[cmd_str = "IntelAMTSDK/Samples/RemoteControl/remoteControl"
 566                 cmd_str = "cmdamt/remoteControl"
 567
 568                 if dryrun:
 569                         # NOTE: -p checks the power state of the host.
 570                         # TODO: parse the output to find out if it's ok or not.
 571                         cmd_str += " -p http://%s:16992/RemoteControlService  -user admin -pass '%s' " % (self.host, self.password )
 572                 else:
 573                         cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
 574
 575                 print cmd_str
 576                 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
 577
 578 class DRACRacAdm(PCUControl):
 579         def run(self, node_port, dryrun):
 580
 581                 print "trying racadm_reboot..."
 582                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 583
 584                 return 0
 585
 586 class DRAC(PCUControl):
 587         def run(self, node_port, dryrun):
 588                 self.open(self.host, self.username)
 589                 self.sendPassword(self.password)
 590
 591                 print "logging in..."
 592                 self.transport.write("\r\n")
 593                 # Testing Reboot ?
 594                 if dryrun:
 595                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 596                 else:
 597                         # Reset this machine
 598                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 599
 600                 self.ifThenSend("[%s]#" % self.username, "exit")
 601
 602                 self.close()
 603                 return 0
 604
 605 class HPiLO(PCUControl):
 606         def run(self, node_port, dryrun):
 607                 self.open(self.host, self.username)
 608                 self.sendPassword(self.password)
 609
 610                 # </>hpiLO->
 611                 self.ifThenSend("</>hpiLO->", "cd system1")
 612
 613                 # Reboot Outlet  N        (Y/N)?
 614                 if dryrun:
 615                         self.ifThenSend("</system1>hpiLO->", "POWER")
 616                 else:
 617                         # Reset this machine
 618                         self.ifThenSend("</system1>hpiLO->", "reset")
 619
 620                 self.ifThenSend("</system1>hpiLO->", "exit")
 621
 622                 self.close()
 623                 return 0
 624
 625
 626 class HPiLOHttps(PCUControl):
 627         def run(self, node_port, dryrun):
 628
 629                 locfg = moncommands.CMD()
 630                 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 631                                         self.host, "iloxml/Get_Network.xml",
 632                                         self.username, self.password)
 633                 sout, serr = locfg.run_noexcept(cmd)
 634
 635                 if sout.strip() != "":
 636                         print "sout: %s" % sout.strip()
 637                         return sout.strip()
 638
 639                 if not dryrun:
 640                         locfg = moncommands.CMD()
 641                         cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 642                                                 self.host, "iloxml/Reset_Server.xml",
 643                                                 self.username, self.password)
 644                         sout, serr = locfg.run_noexcept(cmd)
 645
 646                         if sout.strip() != "":
 647                                 print "sout: %s" % sout.strip()
 648                                 #return sout.strip()
 649                 return 0
 650
 651 class BayTechAU(PCUControl):
 652         def run(self, node_port, dryrun):
 653                 self.open(self.host, self.username, None, "Enter user name:")
 654                 self.sendPassword(self.password, "Enter Password:")
 655
 656                 #self.ifThenSend("RPC-16>", "Status")
 657                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 658
 659                 # Reboot Outlet  N        (Y/N)?
 660                 if dryrun:
 661                         self.ifThenSend("(Y/N)?", "N")
 662                 else:
 663                         self.ifThenSend("(Y/N)?", "Y")
 664                 self.ifThenSend("RPC3-NC>", "")
 665
 666                 self.close()
 667                 return 0
 668
 669 class BayTechGeorgeTown(PCUControl):
 670         def run(self, node_port, dryrun):
 671                 self.open(self.host, self.username, None, "Enter user name:")
 672                 self.sendPassword(self.password, "Enter Password:")
 673
 674                 #self.ifThenSend("RPC-16>", "Status")
 675
 676                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 677
 678                 # Reboot Outlet  N        (Y/N)?
 679                 if dryrun:
 680                         self.ifThenSend("(Y/N)?", "N")
 681                 else:
 682                         self.ifThenSend("(Y/N)?", "Y")
 683                 self.ifThenSend("RPC-16>", "")
 684
 685                 self.close()
 686                 return 0
 687
 688 class BayTechCtrlCUnibe(PCUControl):
 689         """
 690                 For some reason, these units let you log in fine, but they hang
 691                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 692                 why.
 693         """
 694         def run(self, node_port, dryrun):
 695                 print "BayTechCtrlC %s" % self.host
 696
 697                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 698                 s = pxssh.pxssh()
 699                 if not s.login(self.host, self.username, self.password, ssh_options):
 700                         raise ExceptionPassword("Invalid Password")
 701                 # Otherwise, the login succeeded.
 702
 703                 # Send a ctrl-c to the remote process.
 704                 print "sending ctrl-c"
 705                 s.send(chr(3))
 706
 707                 # Control Outlets  (5 ,1).........5
 708                 try:
 709                         index = s.expect(["Enter Request :"])
 710
 711                         if index == 0:
 712                                 print "3"
 713                                 s.send("3\r\n")
 714                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 715                                 if index == 1:
 716                                         s.send(self.username + "\r\n")
 717                                         index = s.expect(["DS-RPC>"])
 718
 719                                 if index == 0:
 720                                         print "Reboot %d" % node_port
 721                                         s.send("Reboot %d\r\n" % node_port)
 722
 723                                         index = s.expect(["(Y/N)?"])
 724                                         if index == 0:
 725                                                 if dryrun:
 726                                                         print "sending N"
 727                                                         s.send("N\r\n")
 728                                                 else:
 729                                                         print "sending Y"
 730                                                         s.send("Y\r\n")
 731
 732                                 #index = s.expect(["DS-RPC>"])
 733                                 #print "got prompt back"
 734
 735                         s.close()
 736
 737                 except pexpect.EOF:
 738                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 739                 except pexpect.TIMEOUT:
 740                         raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
 741
 742                 return 0
 743
 744 class BayTechCtrlC(PCUControl):
 745         """
 746                 For some reason, these units let you log in fine, but they hang
 747                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 748                 why.
 749         """
 750         def run(self, node_port, dryrun):
 751                 print "BayTechCtrlC %s" % self.host
 752
 753                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 754                 s = pxssh.pxssh()
 755                 if not s.login(self.host, self.username, self.password, ssh_options):
 756                         raise ExceptionPassword("Invalid Password")
 757                 # Otherwise, the login succeeded.
 758
 759                 # Send a ctrl-c to the remote process.
 760                 print "sending ctrl-c"
 761                 s.send(chr(3))
 762
 763                 # Control Outlets  (5 ,1).........5
 764                 try:
 765                         index = s.expect(["Enter Request :"])
 766
 767                         if index == 0:
 768                                 print "5"
 769                                 s.send("5\r\n")
 770                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 771                                 if index == 1:
 772                                         print "sending username"
 773                                         s.send(self.username + "\r\n")
 774                                         index = s.expect(["DS-RPC>"])
 775
 776                                 if index == 0:
 777                                         print "Reboot %d" % node_port
 778                                         s.send("Reboot %d\r\n" % node_port)
 779
 780                                         index = s.expect(["(Y/N)?"])
 781                                         if index == 0:
 782                                                 if dryrun:
 783                                                         print "sending N"
 784                                                         s.send("N\r\n")
 785                                                 else:
 786                                                         print "sending Y"
 787                                                         s.send("Y\r\n")
 788
 789                                 index = s.expect(["DS-RPC>"])
 790                                 #print "got prompt back"
 791
 792                         s.close()
 793
 794                 except pexpect.EOF:
 795                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 796                 except pexpect.TIMEOUT:
 797                         raise ExceptionPrompt("Timeout before Prompt")
 798
 799                 return 0
 800
 801 class BayTech(PCUControl):
 802         def run(self, node_port, dryrun):
 803                 self.open(self.host, self.username)
 804                 self.sendPassword(self.password)
 805
 806                 # Control Outlets  (5 ,1).........5
 807                 self.ifThenSend("Enter Request :", "5")
 808
 809                 # Reboot N
 810                 try:
 811                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 812                 except ExceptionNotFound, msg:
 813                         # one machine is configured to ask for a username,
 814                         # even after login...
 815                         print "msg: %s" % msg
 816                         self.transport.write(self.username + "\r\n")
 817                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 818
 819                 # Reboot Outlet  N        (Y/N)?
 820                 if dryrun:
 821                         self.ifThenSend("(Y/N)?", "N")
 822                 else:
 823                         self.ifThenSend("(Y/N)?", "Y")
 824                 self.ifThenSend("DS-RPC>", "")
 825
 826                 self.close()
 827                 return 0
 828
 829 class WTIIPS4(PCUControl):
 830         def run(self, node_port, dryrun):
 831                 self.open(self.host)
 832                 self.sendPassword(self.password, "Enter Password:")
 833
 834                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 835                 if not dryrun:
 836                         self.ifThenSend("Sure? (Y/N): ", "N")
 837                 else:
 838                         self.ifThenSend("Sure? (Y/N): ", "Y")
 839
 840                 self.ifThenSend("IPS> ", "")
 841
 842                 self.close()
 843                 return 0
 844
 845 class ePowerSwitchGood(PCUControl):
 846         # NOTE:
 847         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
 848         #               For some reason this both doesn't work and in some cases, actually
 849         #               hangs the PCU.  Definitely not what we want.
 850         #
 851         #               The code below is much simpler.  Just letting things fail first,
 852         #               and then, trying again with authentication string in the header.
 853         #
 854         def run(self, node_port, dryrun):
 855                 self.transport = None
 856                 self.url = "http://%s:%d/" % (self.host,80)
 857                 uri = "%s:%d" % (self.host,80)
 858
 859                 req = urllib2.Request(self.url)
 860                 try:
 861                         handle = urllib2.urlopen(req)
 862                 except IOError, e:
 863                         # NOTE: this is expected to fail initially
 864                         pass
 865                 else:
 866                         print self.url
 867                         print "-----------"
 868                         print handle.read()
 869                         print "-----------"
 870                         return "ERROR: not protected by HTTP authentication"
 871
 872                 if not hasattr(e, 'code') or e.code != 401:
 873                         return "ERROR: failed for: %s" % str(e)
 874
 875                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
 876                 # NOTE: assuming basic realm authentication.
 877                 authheader = "Basic %s" % base64data
 878                 req.add_header("Authorization", authheader)
 879
 880                 try:
 881                         f = urllib2.urlopen(req)
 882                 except IOError, e:
 883                         # failing here means the User/passwd is wrong (hopefully)
 884                         raise ExceptionPassword("Incorrect username/password")
 885
 886                 # NOTE: after verifying that the user/password is correct,
 887                 #               actually reboot the given node.
 888                 if not dryrun:
 889                         try:
 890                                 data = urllib.urlencode({'P%d' % node_port : "r"})
 891                                 req = urllib2.Request(self.url + "cmd.html")
 892                                 req.add_header("Authorization", authheader)
 893                                 # add data to handler,
 894                                 f = urllib2.urlopen(req, data)
 895                                 if self.verbose: print f.read()
 896                         except:
 897                                 import traceback; traceback.print_exc()
 898
 899                                 # fetch url one more time on cmd.html, econtrol.html or whatever.
 900                                 # pass
 901                 else:
 902                         if self.verbose: print f.read()
 903
 904                 self.close()
 905                 return 0
 906
 907 class CustomPCU(PCUControl):
 908         def run(self, node_port, dryrun):
 909                 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
 910
 911                 if not dryrun:
 912                         # Turn host off, then on
 913                         formstr = "plab%s=off" % node_port
 914                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 915                         time.sleep(5)
 916                         formstr = "plab%s=on" % node_port
 917                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 918                 else:
 919                         os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
 920
 921
 922 class ePowerSwitchOld(PCUControl):
 923         def run(self, node_port, dryrun):
 924                 self.url = "http://%s:%d/" % (self.host,80)
 925                 uri = "%s:%d" % (self.host,80)
 926
 927                 # create authinfo
 928                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 929                 authinfo.add_password (None, uri, self.username, self.password)
 930                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 931
 932                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 933                 transport = urllib2.build_opener(authinfo)
 934                 f = transport.open(self.url)
 935                 if self.verbose: print f.read()
 936
 937                 if not dryrun:
 938                         transport = urllib2.build_opener(authhandler)
 939                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
 940                         if self.verbose: print f.read()
 941
 942                 self.close()
 943                 return 0
 944
 945 class ePowerSwitch(PCUControl):
 946         def run(self, node_port, dryrun):
 947                 self.url = "http://%s:%d/" % (self.host,80)
 948                 uri = "%s:%d" % (self.host,80)
 949
 950                 # TODO: I'm still not sure what the deal is here.
 951                 #               two independent calls appear to need to be made before the
 952                 #               reboot will succeed.  It doesn't seem to be possible to do
 953                 #               this with a single call.  I have no idea why.
 954
 955                 # create authinfo
 956                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 957                 authinfo.add_password (None, uri, self.username, self.password)
 958                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 959
 960                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 961                 transport = urllib2.build_opener()
 962                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
 963                 if self.verbose: print f.read()
 964
 965                 if not dryrun:
 966                         transport = urllib2.build_opener(authhandler)
 967                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
 968                         if self.verbose: print f.read()
 969
 970                 #       data= "P%d=r" % node_port
 971                 #self.open(self.host, self.username, self.password)
 972                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
 973                 #self.sendHTTP("econtrol.html", data)
 974                 #self.sendHTTP("cmd.html", data)
 975
 976                 self.close()
 977                 return 0
 978
 979
 980 ### rebooting european BlackBox PSE boxes
 981 # Thierry Parmentelat - May 11 2005
 982 # tested on 4-ports models known as PSE505-FR
 983 # uses http to POST a data 'P<port>=r'
 984 # relies on basic authentication within http1.0
 985 # first curl-based script was
 986 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
 987 #       http://<hostname>:<http_port>/cmd.html && echo OK
 988
 989 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
 990
 991         global verbose
 992
 993         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
 994         data= "P%d=r" % port_in_pcu
 995         if verbose:
 996                 logger.debug("POSTing '%s' on %s" % (data,url))
 997
 998         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 999         uri = "%s:%d" % (pcu_ip,http_port)
1000         authinfo.add_password (None, uri, username, password)
1001         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1002
1003         opener = urllib2.build_opener(authhandler)
1004         urllib2.install_opener(opener)
1005
1006         if (dryrun):
1007                 return 0
1008
1009         try:
1010                 f = urllib2.urlopen(url,data)
1011
1012                 r= f.read()
1013                 if verbose:
1014                         logger.debug(r)
1015                 return 0
1016
1017         except urllib2.URLError,err:
1018                 logger.info('Could not open http connection', err)
1019                 return "bbpse error"
1020
1021 ### rebooting x10toggle based systems addressed by port
1022 # Marc E. Fiuczynski - May 31 2005
1023 # tested on 4-ports models known as PSE505-FR
1024 # uses ssh and password to login to an account
1025 # that will cause the system to be powercycled.
1026
1027 def x10toggle_reboot(ip, username, password, port, dryrun):
1028         global verbose
1029
1030         ssh = None
1031         try:
1032                 ssh = pyssh.Ssh(username, ip)
1033                 ssh.open()
1034
1035                 # Login
1036                 telnet_answer(ssh, "password:", password)
1037
1038                 if not dryrun:
1039                         # Reboot
1040                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
1041
1042                 # Close
1043                 output = ssh.close()
1044                 if verbose:
1045                         logger.debug(output)
1046                 return 0
1047
1048         except Exception, err:
1049                 if verbose:
1050                         logger.debug(err)
1051                 if ssh:
1052                         output = ssh.close()
1053                         if verbose:
1054                                 logger.debug(output)
1055                 return errno.ETIMEDOUT
1056
1057 ### rebooting Dell systems via RAC card
1058 # Marc E. Fiuczynski - June 01 2005
1059 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1060 #
1061
1062 def runcmd(command, args, username, password, timeout = None):
1063
1064         result = [None]
1065         result_ready = threading.Condition()
1066
1067         def set_result(x):
1068
1069                 result_ready.acquire()
1070                 try:
1071                         result[0] = x
1072                 finally:
1073                         result_ready.notify()
1074                         result_ready.release()
1075
1076         def do_command(command, username, password):
1077
1078                 try:
1079                         # Popen4 is a popen-type class that combines stdout and stderr
1080                         p = popen2.Popen4(command)
1081
1082                         # read all output data
1083                         p.tochild.write("%s\n" % username)
1084                         p.tochild.write("%s\n" % password)
1085                         p.tochild.close()
1086                         data = p.fromchild.read()
1087
1088                         while True:
1089                                 # might get interrupted by a signal in poll() or waitpid()
1090                                 try:
1091                                         retval = p.wait()
1092                                         set_result((retval, data))
1093                                         break
1094                                 except OSError, ex:
1095                                         if ex.errno == errno.EINTR:
1096                                                 continue
1097                                         raise ex
1098                 except Exception, ex:
1099                         set_result(ex)
1100
1101         if args:
1102                 command = " ".join([command] + args)
1103
1104         worker = threading.Thread(target = do_command, args = (command, username, password, ))
1105         worker.setDaemon(True)
1106         result_ready.acquire()
1107         worker.start()
1108         result_ready.wait(timeout)
1109         try:
1110                 if result == [None]:
1111                         raise Exception, "command timed-out: '%s'" % command
1112         finally:
1113                 result_ready.release()
1114         result = result[0]
1115
1116         if isinstance(result, Exception):
1117                 raise result
1118         else:
1119                 (retval, data) = result
1120                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1121                         return data
1122                 else:
1123                         out = "system command ('%s') " % command
1124                         if os.WIFEXITED(retval):
1125                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1126                         else:
1127                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1128                         if data:
1129                                 out += "; output follows:\n" + data
1130                         raise Exception, out
1131
1132 def racadm_reboot(host, username, password, port, dryrun):
1133         global verbose
1134
1135         ip = socket.gethostbyname(host)
1136         try:
1137                 cmd = "/usr/sbin/racadm"
1138                 os.stat(cmd)
1139                 if not dryrun:
1140                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1141                                 username, password)
1142                 else:
1143                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1144                                 username, password)
1145
1146                 print "RUNCMD: %s" % output
1147                 if verbose:
1148                         logger.debug(output)
1149                 return 0
1150
1151         except Exception, err:
1152                 logger.debug("runcmd raised exception %s" % err)
1153                 if verbose:
1154                         logger.debug(err)
1155                 return -1
1156
1157 def pcu_name(pcu):
1158         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1159                 return pcu['hostname']
1160         elif pcu['ip'] is not None and pcu['ip'] is not "":
1161                 return pcu['ip']
1162         else:
1163                 return None
1164
1165 import database
1166 fb =database.dbLoad("findbadpcus")
1167
1168 def get_pcu_values(pcu_id):
1169         # TODO: obviously, this shouldn't be loaded each time...
1170
1171         try:
1172                 values = fb['nodes']["id_%s" % pcu_id]['values']
1173         except:
1174                 values = None
1175
1176         return values
1177
1178 def reboot(nodename):
1179         return reboot_policy(nodename, True, False)
1180
1181 def reboot_policy(nodename, continue_probe, dryrun):
1182         global verbose
1183
1184         pcu = plc.getpcu(nodename)
1185         if not pcu:
1186                 logger.debug("no pcu for %s" % hostname)
1187                 print "no pcu for %s" % hostname
1188                 return False # "%s has no pcu" % nodename
1189
1190         values = get_pcu_values(pcu['pcu_id'])
1191         if values == None:
1192                 logger.debug("No values for pcu probe %s" % hostname)
1193                 print "No values for pcu probe %s" % hostname
1194                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1195
1196         # Try the PCU first
1197         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1198
1199         ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1200
1201         if ret != 0:
1202                 print ret
1203                 return False
1204         else:
1205                 print "return true"
1206                 return True
1207
1208 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1209         rb_ret = ""
1210
1211         try:
1212                 # DataProbe iPal (many sites)
1213                 if  continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1214                         ipal = IPAL(values, verbose, ['23', '80', '9100'])
1215                         rb_ret = ipal.reboot(values[nodename], dryrun)
1216
1217                 # APC Masterswitch (Berkeley)
1218                 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1219                                                                   values['model'].find("Masterswitch") >= 0 ):
1220                         print values
1221
1222                         # TODO: make a more robust version of APC
1223                         if values['pcu_id'] in [1163,1055,1111,1231,1113,1127,1128,1148]:
1224                                 apc = APCEurope(values, verbose, ['22', '23'])
1225                                 rb_ret = apc.reboot(values[nodename], dryrun)
1226
1227                         elif values['pcu_id'] in [1110,86]:
1228                                 apc = APCBrazil(values, verbose, ['22', '23'])
1229                                 rb_ret = apc.reboot(values[nodename], dryrun)
1230
1231                         elif values['pcu_id'] in [1221,1225]:
1232                                 apc = APCBerlin(values, verbose, ['22', '23'])
1233                                 rb_ret = apc.reboot(values[nodename], dryrun)
1234
1235                         elif values['pcu_id'] in [1173,1221,1220]:
1236                                 apc = APCFolsom(values, verbose, ['22', '23'])
1237                                 rb_ret = apc.reboot(values[nodename], dryrun)
1238
1239                         else:
1240                                 apc = APCMaster(values, verbose, ['22', '23'])
1241                                 rb_ret = apc.reboot(values[nodename], dryrun)
1242
1243                 # BayTech DS4-RPC
1244                 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1245                         if values['pcu_id'] in [1237,1052,1209,1002,1008,1041,1013,1022]:
1246                                 # These  require a 'ctrl-c' to be sent...
1247                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1248                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1249
1250                         elif values['pcu_id'] in [93]:
1251                                 baytech = BayTechAU(values, verbose, ['22', '23'])
1252                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1253
1254                         elif values['pcu_id'] in [1057]:
1255                                 # These  require a 'ctrl-c' to be sent...
1256                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1257                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1258
1259                         elif values['pcu_id'] in [1012]:
1260                                 # This pcu sometimes doesn't present the 'Username' prompt,
1261                                 # unless you immediately try again...
1262                                 try:
1263                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1264                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1265                                 except:
1266                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1267                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1268                         else:
1269                                 baytech = BayTech(values, verbose, ['22', '23'])
1270                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1271
1272                 # iLO
1273                 elif continue_probe and values['model'].find("ilo") >= 0:
1274                         try:
1275                                 hpilo = HPiLO(values, verbose, ['22'])
1276                                 rb_ret = hpilo.reboot(0, dryrun)
1277                                 if rb_ret != 0:
1278                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1279                                         rb_ret = hpilo.reboot(0, dryrun)
1280                         except:
1281                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1282                                 rb_ret = hpilo.reboot(0, dryrun)
1283
1284                 # DRAC ssh
1285                 elif continue_probe and values['model'].find("DRAC") >= 0:
1286                         # TODO: I don't think DRACRacAdm will throw an exception for the
1287                         # default method to catch...
1288                         try:
1289                                 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1290                                 rb_ret = drac.reboot(0, dryrun)
1291                         except:
1292                                 drac = DRAC(values, verbose, ['22'])
1293                                 rb_ret = drac.reboot(0, dryrun)
1294
1295                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1296                                 wti = WTIIPS4(values, verbose, ['23'])
1297                                 rb_ret = wti.reboot(values[nodename], dryrun)
1298
1299                 elif continue_probe and values['model'].find("AMT") >= 0:
1300                                 amt = IntelAMT(values, verbose, ['16992'])
1301                                 rb_ret = amt.reboot(values[nodename], dryrun)
1302
1303                 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1304                 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1305                         # TODO: allow a different port than http 80.
1306                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1307                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1308                         elif values['pcu_id'] in [1003]:
1309                                 # OLD EPOWER
1310                                 print "OLD EPOWER"
1311                                 eps = ePowerSwitch(values, verbose, ['80'])
1312                         else:
1313                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1314
1315                         rb_ret = eps.reboot(values[nodename], dryrun)
1316                 elif continue_probe and values['pcu_id'] in [1122]:
1317                         custom = CustomPCU(values, verbose, ['80', '443'])
1318                         custom.reboot(values[nodename], dryrun)
1319
1320                 elif continue_probe:
1321                         rb_ret = "Unsupported_PCU"
1322
1323                 elif continue_probe == False:
1324                         if 'portstatus' in values:
1325                                 rb_ret = "NetDown"
1326                         else:
1327                                 rb_ret = "Not_Run"
1328                 else:
1329                         rb_ret = -1
1330
1331         except ExceptionPort, err:
1332                 rb_ret = str(err)
1333
1334         return rb_ret
1335         # ????
1336         #elif continue_probe and values['protocol'] == "racadm" and \
1337         #               values['model'] == "RAC":
1338         #       rb_ret = racadm_reboot(pcu_name(values),
1339         #                                                                 values['username'],
1340         #                                                                 values['password'],
1341         #                                                                 pcu[nodename],
1342         #                                                                 dryrun)
1343
1344 def main():
1345         logger.setLevel(logging.DEBUG)
1346         ch = logging.StreamHandler()
1347         ch.setLevel(logging.DEBUG)
1348         formatter = logging.Formatter('LOGGER - %(message)s')
1349         ch.setFormatter(formatter)
1350         logger.addHandler(ch)
1351
1352         try:
1353                 if "test" in sys.argv:
1354                         dryrun = True
1355                 else:
1356                         dryrun = False
1357
1358                 for node in sys.argv[1:]:
1359                         if node == "test": continue
1360
1361                         print "Rebooting %s" % node
1362                         if reboot_policy(node, True, dryrun):
1363                                 print "success"
1364                         else:
1365                                 print "failed"
1366         except Exception, err:
1367                 import traceback; traceback.print_exc()
1368                 print err
1369
1370 if __name__ == '__main__':
1371         import plc
1372         logger = logging.getLogger("monitor")
1373         main()