reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import urllib
  12 import threading, popen2
  13 import array, struct
  14 #from socket import *
  15 import socket
  16 import plc
  17 import base64
  18 from subprocess import PIPE, Popen
  19 import ssh.pxssh as pxssh
  20 import ssh.pexpect as pexpect
  21 import socket
  22 import moncommands
  23
  24 # Use our versions of telnetlib and pyssh
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  26 import telnetlib
  27 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  28 import pyssh
  29
  30 # Timeouts in seconds
  31 TELNET_TIMEOUT = 45
  32
  33 # Event class ID from pcu events
  34 #NODE_POWER_CONTROL = 3
  35
  36 # Monitor user ID
  37 #MONITOR_USER_ID = 11142
  38
  39 import logging
  40 logger = logging.getLogger("monitor")
  41 verbose = 1
  42 #dryrun = 0;
  43
  44 class ExceptionNoTransport(Exception): pass
  45 class ExceptionNotFound(Exception): pass
  46 class ExceptionPassword(Exception): pass
  47 class ExceptionTimeout(Exception): pass
  48 class ExceptionPrompt(Exception): pass
  49 class ExceptionSequence(Exception): pass
  50 class ExceptionReset(Exception): pass
  51 class ExceptionPort(Exception): pass
  52 class ExceptionUsername(Exception): pass
  53
  54 def telnet_answer(telnet, expected, buffer):
  55         global verbose
  56
  57         output = telnet.read_until(expected, TELNET_TIMEOUT)
  58         #if verbose:
  59         #       logger.debug(output)
  60         if output.find(expected) == -1:
  61                 raise ExceptionNotFound, "'%s' not found" % expected
  62         else:
  63                 telnet.write(buffer + "\r\n")
  64
  65
  66 # PCU has model, host, preferred-port, user, passwd,
  67
  68 # This is an object derived directly form the PLCAPI DB fields
  69 class PCU(object):
  70         def __init__(self, plc_pcu_dict):
  71                 for field in ['username', 'password', 'site_id',
  72                                                 'hostname', 'ip',
  73                                                 'pcu_id', 'model',
  74                                                 'node_ids', 'ports', ]:
  75                         if field in plc_pcu_dict:
  76                                 self.__setattr__(field, plc_pcu_dict[field])
  77                         else:
  78                                 raise Exception("No such field %s in PCU object" % field)
  79
  80 # These are the convenience functions build around the PCU object.
  81 class PCUModel(PCU):
  82         def __init__(self, plc_pcu_dict):
  83                 PCU.__init__(self, plc_pcu_dict)
  84                 self.host = self.pcu_name()
  85
  86         def pcu_name(self):
  87                 if self.hostname is not None and self.hostname is not "":
  88                         return self.hostname
  89                 elif self.ip is not None and self.ip is not "":
  90                         return self.ip
  91                 else:
  92                         return None
  93
  94         def nodeidToPort(self, node_id):
  95                 if node_id in self.node_ids:
  96                         for i in range(0, len(self.node_ids)):
  97                                 if node_id == self.node_ids[i]:
  98                                         return self.ports[i]
  99
 100                 raise Exception("No such Node ID: %d" % node_id)
 101
 102 # This class captures the observed pcu records from FindBadPCUs.py
 103 class PCURecord:
 104         def __init__(self, pcu_record_dict):
 105                 for field in ['nodenames', 'portstatus',
 106                                                 'dnsmatch',
 107                                                 'complete_entry', ]:
 108                         if field in pcu_record_dict:
 109                                 if field == "reboot":
 110                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 111                                 else:
 112                                         self.__setattr__(field, pcu_record_dict[field])
 113                         else:
 114                                 raise Exception("No such field %s in pcu record dict" % field)
 115
 116 class Transport:
 117         TELNET = 1
 118         SSH    = 2
 119         HTTP   = 3
 120         IPAL   = 4
 121
 122         TELNET_TIMEOUT = 60
 123
 124         def __init__(self, type, verbose):
 125                 self.type = type
 126                 self.verbose = verbose
 127                 self.transport = None
 128
 129         def open(self, host, username=None, password=None, prompt="User Name"):
 130                 transport = None
 131
 132                 if self.type == self.TELNET:
 133                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 134                         transport.set_debuglevel(self.verbose)
 135                         if username is not None:
 136                                 self.transport = transport
 137                                 self.ifThenSend(prompt, username, ExceptionUsername)
 138
 139                 elif self.type == self.SSH:
 140                         if username is not None:
 141                                 transport = pyssh.Ssh(username, host)
 142                                 transport.set_debuglevel(self.verbose)
 143                                 transport.open()
 144                                 # TODO: have an ssh set_debuglevel() also...
 145                         else:
 146                                 raise Exception("Username cannot be None for ssh transport.")
 147                 elif self.type == self.HTTP:
 148                         self.url = "http://%s:%d/" % (host,80)
 149                         uri = "%s:%d" % (host,80)
 150
 151                         # create authinfo
 152                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 153                         authinfo.add_password (None, uri, username, password)
 154                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 155
 156                         transport = urllib2.build_opener(authhandler)
 157
 158                 else:
 159                         raise Exception("Unknown transport type: %s" % self.type)
 160
 161                 self.transport = transport
 162                 return True
 163
 164         def close(self):
 165                 if self.type == self.TELNET:
 166                         self.transport.close()
 167                 elif self.type == self.SSH:
 168                         self.transport.close()
 169                 elif self.type == self.HTTP:
 170                         pass
 171                 else:
 172                         raise Exception("Unknown transport type %s" % self.type)
 173                 self.transport = None
 174
 175         def sendHTTP(self, resource, data):
 176                 if self.verbose:
 177                         print "POSTing '%s' to %s" % (data,self.url + resource)
 178
 179                 try:
 180                         f = self.transport.open(self.url + resource ,data)
 181                         r = f.read()
 182                         if self.verbose:
 183                                 print r
 184
 185                 except urllib2.URLError,err:
 186                         logger.info('Could not open http connection', err)
 187                         return "http transport error"
 188
 189                 return 0
 190
 191         def sendPassword(self, password, prompt=None):
 192                 if self.type == self.TELNET:
 193                         if prompt == None:
 194                                 self.ifThenSend("Password", password, ExceptionPassword)
 195                         else:
 196                                 self.ifThenSend(prompt, password, ExceptionPassword)
 197                 elif self.type == self.SSH:
 198                         self.ifThenSend("password:", password, ExceptionPassword)
 199                 elif self.type == self.HTTP:
 200                         pass
 201                 else:
 202                         raise Exception("Unknown transport type: %s" % self.type)
 203
 204         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 205
 206                 if self.transport != None:
 207                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 208                         if output.find(expected) == -1:
 209                                 raise ErrorClass, "'%s' not found" % expected
 210                         else:
 211                                 self.transport.write(buffer + "\r\n")
 212                 else:
 213                         raise ExceptionNoTransport("transport object is type None")
 214
 215         def ifElse(self, expected, ErrorClass):
 216                 try:
 217                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 218                 except:
 219                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 220
 221
 222 class PCUControl(Transport,PCUModel,PCURecord):
 223         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 224                 PCUModel.__init__(self, plc_pcu_record)
 225                 PCURecord.__init__(self, plc_pcu_record)
 226                 type = None
 227                 if self.portstatus:
 228                         if '22' in supported_ports and self.portstatus['22'] == "open":
 229                                 type = Transport.SSH
 230                         elif '23' in supported_ports and self.portstatus['23'] == "open":
 231                                 type = Transport.TELNET
 232                         elif '80' in supported_ports and self.portstatus['80'] == "open":
 233                                 type = Transport.HTTP
 234                         elif '443' in supported_ports and self.portstatus['443'] == "open":
 235                                 type = Transport.HTTP
 236                         elif '5869' in supported_ports and self.portstatus['5869'] == "open":
 237                                 # For DRAC cards. Racadm opens this port.
 238                                 type = Transport.HTTP
 239                         elif '9100' in supported_ports and self.portstatus['9100'] == "open":
 240                                 type = Transport.IPAL
 241                         elif '16992' in supported_ports and self.portstatus['16992'] == "open":
 242                                 type = Transport.HTTP
 243                         else:
 244                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 245                 else:
 246                         raise Exception("No Portstatus: No transport because no open ports")
 247                 Transport.__init__(self, type, verbose)
 248
 249         def run(self, node_port, dryrun):
 250                 """ This function is to be defined by the specific PCU instance.  """
 251                 pass
 252
 253         def reboot(self, node_port, dryrun):
 254                 try:
 255                         return self.run(node_port, dryrun)
 256                 except ExceptionNotFound, err:
 257                         return "error: " + str(err)
 258                 except ExceptionPassword, err:
 259                         return "password exception: " + str(err)
 260                 except ExceptionTimeout, err:
 261                         return "timeout exception: " + str(err)
 262                 except ExceptionUsername, err:
 263                         return "exception: no username prompt: " + str(err)
 264                 except ExceptionSequence, err:
 265                         return "sequence error: " + str(err)
 266                 except ExceptionPrompt, err:
 267                         return "prompt exception: " + str(err)
 268                 except ExceptionPort, err:
 269                         return "no ports exception: " + str(err)
 270                 except socket.error, err:
 271                         return "socket error: timeout: " + str(err)
 272                 except EOFError, err:
 273                         if self.verbose:
 274                                 logger.debug("reboot: EOF")
 275                                 logger.debug(err)
 276                         self.transport.close()
 277                         import traceback
 278                         traceback.print_exc()
 279                         return "EOF connection reset" + str(err)
 280
 281 class IPAL(PCUControl):
 282         """
 283                 This now uses a proprietary format for communicating with the PCU.  I
 284                 prefer it to Telnet, and Web access, since it's much lighter weight
 285                 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
 286         """
 287
 288         def format_msg(self, data, cmd):
 289                 esc = chr(int('1b',16))
 290                 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
 291
 292         def recv_noblock(self, s, count):
 293                 import errno
 294
 295                 try:
 296                         # TODO: make sleep backoff, before stopping.
 297                         time.sleep(4)
 298                         ret = s.recv(count, socket.MSG_DONTWAIT)
 299                 except socket.error, e:
 300                         if e[0] == errno.EAGAIN:
 301                                 raise Exception(e[1])
 302                         else:
 303                                 # TODO: not other exceptions.
 304                                 raise Exception(e)
 305                 return ret
 306
 307         def run(self, node_port, dryrun):
 308                 import errno
 309
 310                 power_on = False
 311
 312                 print "open socket"
 313                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 314                 try:
 315                         print "connect"
 316                         s.connect((self.host, 9100))
 317                 except socket.error, e:
 318                         s.close()
 319                         if e[0] == errno.ECONNREFUSED:
 320                                 # cannot connect to remote host
 321                                 raise Exception(e[1])
 322                         else:
 323                                 # TODO: what other conditions are there?
 324                                 raise Exception(e)
 325
 326                 # get current status
 327                 print "Checking status"
 328                 s.send(self.format_msg("", 'O'))
 329                 ret = self.recv_noblock(s, 8)
 330                 print "Current status is '%s'" % ret
 331
 332                 if ret == '':
 333                         raise Exception("Status returned 'another session already open' %s : %s" % (node_port, ret))
 334
 335
 336                 if node_port < len(ret):
 337                         status = ret[node_port]
 338                         if status == '1':
 339                                 # up
 340                                 power_on = True
 341                         elif status == '0':
 342                                 # down
 343                                 power_on = False
 344                         else:
 345                                 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 346                 else:
 347                         raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 348
 349
 350                 if not dryrun:
 351                         if power_on:
 352                                 print "Pulsing %s" % node_port
 353                                 s.send(self.format_msg("%s" % node_port, 'P'))
 354                         else:
 355                                 # NOTE: turn power on ; do not pulse the port.
 356                                 print "Power was off, so turning on ..."
 357                                 s.send(self.format_msg("%s" % node_port, 'E'))
 358                                 #s.send(self.format_msg("%s" % node_port, 'P'))
 359
 360                         print "Receiving response."
 361                         ret = self.recv_noblock(s, 8)
 362                         print "Current status is '%s'" % ret
 363
 364                         if node_port < len(ret):
 365                                 status = ret[node_port]
 366                                 if status == '1':
 367                                         # up
 368                                         power_on = True
 369                                 elif status == '0':
 370                                         # down
 371                                         power_on = False
 372                                 else:
 373                                         raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 374                         else:
 375                                 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 376
 377                         if power_on:
 378                                 return 0
 379                         else:
 380                                 return "Failed Power On"
 381
 382                 s.close()
 383                 return 0
 384
 385 # TELNET version of protocol...
 386 #               #self.open(self.host)
 387 #               ## XXX Some iPals require you to hit Enter a few times first
 388 #               #self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 389 #               # Login
 390 #               self.ifThenSend("Password >", self.password, ExceptionPassword)
 391 #               self.transport.write("\r\n\r\n")
 392 #               if not dryrun: # P# - Pulse relay
 393 #                       print "node_port %s" % node_port
 394 #                       self.ifThenSend("Enter >",
 395 #                                                       "P7", # % node_port,
 396 #                                                       ExceptionNotFound)
 397 #                       print "send newlines"
 398 #                       self.transport.write("\r\n\r\n")
 399 #                       print "after new lines"
 400 #               # Get the next prompt
 401 #               print "wait for enter"
 402 #               self.ifElse("Enter >", ExceptionTimeout)
 403 #               print "closing "
 404 #               self.close()
 405 #               return 0
 406
 407 class APCEurope(PCUControl):
 408         def run(self, node_port, dryrun):
 409                 self.open(self.host, self.username)
 410                 self.sendPassword(self.password)
 411
 412                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 413                 self.ifThenSend("\r\n> ", "2")
 414                 self.ifThenSend("\r\n> ", str(node_port))
 415                 # 3- Immediate Reboot
 416                 self.ifThenSend("\r\n> ", "3")
 417
 418                 if not dryrun:
 419                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 420                                                         "YES\r\n",
 421                                                         ExceptionSequence)
 422                 else:
 423                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 424                                                         "", ExceptionSequence)
 425                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 426
 427                 self.close()
 428                 return 0
 429
 430 class APCBrazil(PCUControl):
 431         def run(self, node_port, dryrun):
 432                 self.open(self.host, self.username)
 433                 self.sendPassword(self.password)
 434
 435                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 436                 self.ifThenSend("\r\n> ", str(node_port))
 437                 # 4- Immediate Reboot
 438                 self.ifThenSend("\r\n> ", "4")
 439
 440                 if not dryrun:
 441                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 442                                                         "YES\r\n",
 443                                                         ExceptionSequence)
 444                 else:
 445                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 446                                                         "", ExceptionSequence)
 447                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 448
 449                 self.close()
 450                 return 0
 451
 452 class APCBerlin(PCUControl):
 453         def run(self, node_port, dryrun):
 454                 self.open(self.host, self.username)
 455                 self.sendPassword(self.password)
 456
 457                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 458                 self.ifThenSend("\r\n> ", "2")
 459                 self.ifThenSend("\r\n> ", "1")
 460                 self.ifThenSend("\r\n> ", str(node_port))
 461                 # 3- Immediate Reboot
 462                 self.ifThenSend("\r\n> ", "3")
 463
 464                 if not dryrun:
 465                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 466                                                         "YES\r\n",
 467                                                         ExceptionSequence)
 468                 else:
 469                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 470                                                         "", ExceptionSequence)
 471                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 472
 473                 self.close()
 474                 return 0
 475
 476 class APCFolsom(PCUControl):
 477         def run(self, node_port, dryrun):
 478                 self.open(self.host, self.username)
 479                 self.sendPassword(self.password)
 480
 481                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 482                 self.ifThenSend("\r\n> ", "2")
 483                 self.ifThenSend("\r\n> ", "1")
 484                 self.ifThenSend("\r\n> ", str(node_port))
 485                 self.ifThenSend("\r\n> ", "1")
 486
 487                 # 3- Immediate Reboot
 488                 self.ifThenSend("\r\n> ", "3")
 489
 490                 if not dryrun:
 491                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 492                                                         "YES\r\n",
 493                                                         ExceptionSequence)
 494                 else:
 495                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 496                                                         "", ExceptionSequence)
 497                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 498
 499                 self.close()
 500                 return 0
 501
 502 class APCMaster(PCUControl):
 503         def run(self, node_port, dryrun):
 504                 print "Rebooting %s" % self.host
 505                 self.open(self.host, self.username)
 506                 self.sendPassword(self.password)
 507
 508                 # 1- Device Manager
 509                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 510                 # 3- Outlet Control/Config
 511                 self.ifThenSend("\r\n> ", "3")
 512                 # n- Outlet n
 513                 self.ifThenSend("\r\n> ", str(node_port))
 514                 # 1- Control Outlet
 515                 self.ifThenSend("\r\n> ", "1")
 516                 # 3- Immediate Reboot
 517                 self.ifThenSend("\r\n> ", "3")
 518
 519                 if not dryrun:
 520                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 521                                                         "YES\r\n",
 522                                                         ExceptionSequence)
 523                 else:
 524                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 525                                                         "", ExceptionSequence)
 526                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 527
 528                 self.close()
 529                 return 0
 530
 531 class APC(PCUControl):
 532         def __init__(self, plc_pcu_record, verbose):
 533                 PCUControl.__init__(self, plc_pcu_record, verbose)
 534
 535                 self.master = APCMaster(plc_pcu_record, verbose)
 536                 self.folsom = APCFolsom(plc_pcu_record, verbose)
 537                 self.europe = APCEurope(plc_pcu_record, verbose)
 538
 539         def run(self, node_port, dryrun):
 540                 try_again = True
 541                 sleep_time = 1
 542
 543                 for pcu in [self.master, self.europe, self.folsom]:
 544                         if try_again:
 545                                 try:
 546                                         print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
 547                                         try_again = False
 548                                         print "sleeping 5"
 549                                         time.sleep(sleep_time)
 550                                         ret = pcu.reboot(node_port, dryrun)
 551                                 except ExceptionSequence, err:
 552                                         del pcu
 553                                         sleep_time = 130
 554                                         try_again = True
 555
 556                 if try_again:
 557                         return "Unknown reboot sequence for APC PCU"
 558                 else:
 559                         return ret
 560
 561 class IntelAMT(PCUControl):
 562         def run(self, node_port, dryrun):
 563
 564                 cmd = moncommands.CMD()
 565                 #[cmd_str = "IntelAMTSDK/Samples/RemoteControl/remoteControl"
 566                 cmd_str = "cmdamt/remoteControl"
 567
 568                 if dryrun:
 569                         # NOTE: -p checks the power state of the host.
 570                         # TODO: parse the output to find out if it's ok or not.
 571                         cmd_str += " -p http://%s:16992/RemoteControlService  -user admin -pass '%s' " % (self.host, self.password )
 572                 else:
 573                         cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
 574
 575                 print cmd_str
 576                 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
 577
 578 class DRACRacAdm(PCUControl):
 579         def run(self, node_port, dryrun):
 580
 581                 print "trying racadm_reboot..."
 582                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 583
 584                 return 0
 585
 586 class DRAC(PCUControl):
 587         def run(self, node_port, dryrun):
 588                 self.open(self.host, self.username)
 589                 self.sendPassword(self.password)
 590
 591                 print "logging in..."
 592                 self.transport.write("\r\n")
 593                 # Testing Reboot ?
 594                 if dryrun:
 595                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 596                 else:
 597                         # Reset this machine
 598                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 599
 600                 self.ifThenSend("[%s]#" % self.username, "exit")
 601
 602                 self.close()
 603                 return 0
 604
 605 class HPiLO(PCUControl):
 606         def run(self, node_port, dryrun):
 607                 self.open(self.host, self.username)
 608                 self.sendPassword(self.password)
 609
 610                 # </>hpiLO->
 611                 self.ifThenSend("</>hpiLO->", "cd system1")
 612
 613                 # Reboot Outlet  N        (Y/N)?
 614                 if dryrun:
 615                         self.ifThenSend("</system1>hpiLO->", "POWER")
 616                 else:
 617                         # Reset this machine
 618                         self.ifThenSend("</system1>hpiLO->", "reset")
 619
 620                 self.ifThenSend("</system1>hpiLO->", "exit")
 621
 622                 self.close()
 623                 return 0
 624
 625
 626 class HPiLOHttps(PCUControl):
 627         def run(self, node_port, dryrun):
 628
 629                 locfg = moncommands.CMD()
 630                 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 631                                         self.host, "iloxml/Get_Network.xml",
 632                                         self.username, self.password)
 633                 sout, serr = locfg.run_noexcept(cmd)
 634
 635                 if sout.strip() != "":
 636                         print "sout: %s" % sout.strip()
 637                         return sout.strip()
 638
 639                 if not dryrun:
 640                         locfg = moncommands.CMD()
 641                         cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 642                                                 self.host, "iloxml/Reset_Server.xml",
 643                                                 self.username, self.password)
 644                         sout, serr = locfg.run_noexcept(cmd)
 645
 646                         if sout.strip() != "":
 647                                 print "sout: %s" % sout.strip()
 648                                 #return sout.strip()
 649                 return 0
 650
 651 class BayTechAU(PCUControl):
 652         def run(self, node_port, dryrun):
 653                 self.open(self.host, self.username, None, "Enter user name:")
 654                 self.sendPassword(self.password, "Enter Password:")
 655
 656                 #self.ifThenSend("RPC-16>", "Status")
 657                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 658
 659                 # Reboot Outlet  N        (Y/N)?
 660                 if dryrun:
 661                         self.ifThenSend("(Y/N)?", "N")
 662                 else:
 663                         self.ifThenSend("(Y/N)?", "Y")
 664                 self.ifThenSend("RPC3-NC>", "")
 665
 666                 self.close()
 667                 return 0
 668
 669 class BayTechGeorgeTown(PCUControl):
 670         def run(self, node_port, dryrun):
 671                 self.open(self.host, self.username, None, "Enter user name:")
 672                 self.sendPassword(self.password, "Enter Password:")
 673
 674                 #self.ifThenSend("RPC-16>", "Status")
 675
 676                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 677
 678                 # Reboot Outlet  N        (Y/N)?
 679                 if dryrun:
 680                         self.ifThenSend("(Y/N)?", "N")
 681                 else:
 682                         self.ifThenSend("(Y/N)?", "Y")
 683                 self.ifThenSend("RPC-16>", "")
 684
 685                 self.close()
 686                 return 0
 687
 688 class BayTechCtrlCUnibe(PCUControl):
 689         """
 690                 For some reason, these units let you log in fine, but they hang
 691                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 692                 why.
 693         """
 694         def run(self, node_port, dryrun):
 695                 print "BayTechCtrlC %s" % self.host
 696
 697                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 698                 s = pxssh.pxssh()
 699                 if not s.login(self.host, self.username, self.password, ssh_options):
 700                         raise ExceptionPassword("Invalid Password")
 701                 # Otherwise, the login succeeded.
 702
 703                 # Send a ctrl-c to the remote process.
 704                 print "sending ctrl-c"
 705                 s.send(chr(3))
 706
 707                 # Control Outlets  (5 ,1).........5
 708                 try:
 709                         print s
 710                         print "Enter Request" in s.before
 711                         index = s.expect("Enter Request")
 712
 713                         if index == 0:
 714                                 print "3"
 715                                 s.send("3\r\n")
 716                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 717                                 if index == 1:
 718                                         s.send(self.username + "\r\n")
 719                                         index = s.expect(["DS-RPC>"])
 720
 721                                 if index == 0:
 722                                         print "Reboot %d" % node_port
 723                                         s.send("Reboot %d\r\n" % node_port)
 724
 725                                         index = s.expect(["(Y/N)?"])
 726                                         if index == 0:
 727                                                 if dryrun:
 728                                                         print "sending N"
 729                                                         s.send("N\r\n")
 730                                                 else:
 731                                                         print "sending Y"
 732                                                         s.send("Y\r\n")
 733
 734                                 #index = s.expect(["DS-RPC>"])
 735                                 #print "got prompt back"
 736
 737                         s.close()
 738
 739                 except pexpect.EOF:
 740                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 741                 except pexpect.TIMEOUT:
 742                         raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
 743
 744                 return 0
 745
 746 class BayTechCtrlC(PCUControl):
 747         """
 748                 For some reason, these units let you log in fine, but they hang
 749                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 750                 why.
 751         """
 752         def run(self, node_port, dryrun):
 753                 print "BayTechCtrlC %s" % self.host
 754
 755                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 756                 s = pxssh.pxssh()
 757                 if not s.login(self.host, self.username, self.password, ssh_options):
 758                         raise ExceptionPassword("Invalid Password")
 759                 # Otherwise, the login succeeded.
 760
 761                 # Send a ctrl-c to the remote process.
 762                 print "sending ctrl-c"
 763                 s.send(chr(3))
 764
 765                 # Control Outlets  (5 ,1).........5
 766                 try:
 767                         index = s.expect(["Enter Request :"])
 768
 769                         if index == 0:
 770                                 print "5"
 771                                 s.send("5\r\n")
 772                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 773                                 if index == 1:
 774                                         print "sending username"
 775                                         s.send(self.username + "\r\n")
 776                                         index = s.expect(["DS-RPC>"])
 777
 778                                 if index == 0:
 779                                         print "Reboot %d" % node_port
 780                                         s.send("Reboot %d\r\n" % node_port)
 781
 782                                         index = s.expect(["(Y/N)?"])
 783                                         if index == 0:
 784                                                 if dryrun:
 785                                                         print "sending N"
 786                                                         s.send("N\r\n")
 787                                                 else:
 788                                                         print "sending Y"
 789                                                         s.send("Y\r\n")
 790
 791                                 # NOTE: for some reason, the script times out with the
 792                                 # following line.  In manual tests, it works correctly, but
 793                                 # with automated tests, evidently it fails.
 794                                 #index = s.expect(["DS-RPC>"])
 795                                 #print "got prompt back"
 796
 797                         s.close()
 798
 799                 except pexpect.EOF:
 800                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 801                 except pexpect.TIMEOUT:
 802                         raise ExceptionPrompt("Timeout before Prompt")
 803
 804                 return 0
 805
 806 class BayTech(PCUControl):
 807         def run(self, node_port, dryrun):
 808                 self.open(self.host, self.username)
 809                 self.sendPassword(self.password)
 810
 811                 # Control Outlets  (5 ,1).........5
 812                 self.ifThenSend("Enter Request :", "5")
 813
 814                 # Reboot N
 815                 try:
 816                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 817                 except ExceptionNotFound, msg:
 818                         # one machine is configured to ask for a username,
 819                         # even after login...
 820                         print "msg: %s" % msg
 821                         self.transport.write(self.username + "\r\n")
 822                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 823
 824                 # Reboot Outlet  N        (Y/N)?
 825                 if dryrun:
 826                         self.ifThenSend("(Y/N)?", "N")
 827                 else:
 828                         self.ifThenSend("(Y/N)?", "Y")
 829                 self.ifThenSend("DS-RPC>", "")
 830
 831                 self.close()
 832                 return 0
 833
 834 class WTIIPS4(PCUControl):
 835         def run(self, node_port, dryrun):
 836                 self.open(self.host)
 837                 self.sendPassword(self.password, "Enter Password:")
 838
 839                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 840                 if not dryrun:
 841                         self.ifThenSend("Sure? (Y/N): ", "N")
 842                 else:
 843                         self.ifThenSend("Sure? (Y/N): ", "Y")
 844
 845                 self.ifThenSend("IPS> ", "")
 846
 847                 self.close()
 848                 return 0
 849
 850 class ePowerSwitchGood(PCUControl):
 851         # NOTE:
 852         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
 853         #               For some reason this both doesn't work and in some cases, actually
 854         #               hangs the PCU.  Definitely not what we want.
 855         #
 856         #               The code below is much simpler.  Just letting things fail first,
 857         #               and then, trying again with authentication string in the header.
 858         #
 859         def run(self, node_port, dryrun):
 860                 self.transport = None
 861                 self.url = "http://%s:%d/" % (self.host,80)
 862                 uri = "%s:%d" % (self.host,80)
 863
 864                 req = urllib2.Request(self.url)
 865                 try:
 866                         handle = urllib2.urlopen(req)
 867                 except IOError, e:
 868                         # NOTE: this is expected to fail initially
 869                         pass
 870                 else:
 871                         print self.url
 872                         print "-----------"
 873                         print handle.read()
 874                         print "-----------"
 875                         return "ERROR: not protected by HTTP authentication"
 876
 877                 if not hasattr(e, 'code') or e.code != 401:
 878                         return "ERROR: failed for: %s" % str(e)
 879
 880                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
 881                 # NOTE: assuming basic realm authentication.
 882                 authheader = "Basic %s" % base64data
 883                 req.add_header("Authorization", authheader)
 884
 885                 try:
 886                         f = urllib2.urlopen(req)
 887                 except IOError, e:
 888                         # failing here means the User/passwd is wrong (hopefully)
 889                         raise ExceptionPassword("Incorrect username/password")
 890
 891                 # NOTE: after verifying that the user/password is correct,
 892                 #               actually reboot the given node.
 893                 if not dryrun:
 894                         try:
 895                                 data = urllib.urlencode({'P%d' % node_port : "r"})
 896                                 req = urllib2.Request(self.url + "cmd.html")
 897                                 req.add_header("Authorization", authheader)
 898                                 # add data to handler,
 899                                 f = urllib2.urlopen(req, data)
 900                                 if self.verbose: print f.read()
 901                         except:
 902                                 import traceback; traceback.print_exc()
 903
 904                                 # fetch url one more time on cmd.html, econtrol.html or whatever.
 905                                 # pass
 906                 else:
 907                         if self.verbose: print f.read()
 908
 909                 self.close()
 910                 return 0
 911
 912 class CustomPCU(PCUControl):
 913         def run(self, node_port, dryrun):
 914                 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
 915
 916                 if not dryrun:
 917                         # Turn host off, then on
 918                         formstr = "plab%s=off" % node_port
 919                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 920                         time.sleep(5)
 921                         formstr = "plab%s=on" % node_port
 922                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 923                 else:
 924                         os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
 925
 926
 927 class ePowerSwitchOld(PCUControl):
 928         def run(self, node_port, dryrun):
 929                 self.url = "http://%s:%d/" % (self.host,80)
 930                 uri = "%s:%d" % (self.host,80)
 931
 932                 # create authinfo
 933                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 934                 authinfo.add_password (None, uri, self.username, self.password)
 935                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 936
 937                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 938                 transport = urllib2.build_opener(authinfo)
 939                 f = transport.open(self.url)
 940                 if self.verbose: print f.read()
 941
 942                 if not dryrun:
 943                         transport = urllib2.build_opener(authhandler)
 944                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
 945                         if self.verbose: print f.read()
 946
 947                 self.close()
 948                 return 0
 949
 950 class ePowerSwitch(PCUControl):
 951         def run(self, node_port, dryrun):
 952                 self.url = "http://%s:%d/" % (self.host,80)
 953                 uri = "%s:%d" % (self.host,80)
 954
 955                 # TODO: I'm still not sure what the deal is here.
 956                 #               two independent calls appear to need to be made before the
 957                 #               reboot will succeed.  It doesn't seem to be possible to do
 958                 #               this with a single call.  I have no idea why.
 959
 960                 # create authinfo
 961                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 962                 authinfo.add_password (None, uri, self.username, self.password)
 963                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 964
 965                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 966                 transport = urllib2.build_opener()
 967                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
 968                 if self.verbose: print f.read()
 969
 970                 if not dryrun:
 971                         transport = urllib2.build_opener(authhandler)
 972                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
 973                         if self.verbose: print f.read()
 974
 975                 #       data= "P%d=r" % node_port
 976                 #self.open(self.host, self.username, self.password)
 977                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
 978                 #self.sendHTTP("econtrol.html", data)
 979                 #self.sendHTTP("cmd.html", data)
 980
 981                 self.close()
 982                 return 0
 983
 984
 985 ### rebooting european BlackBox PSE boxes
 986 # Thierry Parmentelat - May 11 2005
 987 # tested on 4-ports models known as PSE505-FR
 988 # uses http to POST a data 'P<port>=r'
 989 # relies on basic authentication within http1.0
 990 # first curl-based script was
 991 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
 992 #       http://<hostname>:<http_port>/cmd.html && echo OK
 993
 994 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
 995
 996         global verbose
 997
 998         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
 999         data= "P%d=r" % port_in_pcu
1000         if verbose:
1001                 logger.debug("POSTing '%s' on %s" % (data,url))
1002
1003         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1004         uri = "%s:%d" % (pcu_ip,http_port)
1005         authinfo.add_password (None, uri, username, password)
1006         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1007
1008         opener = urllib2.build_opener(authhandler)
1009         urllib2.install_opener(opener)
1010
1011         if (dryrun):
1012                 return 0
1013
1014         try:
1015                 f = urllib2.urlopen(url,data)
1016
1017                 r= f.read()
1018                 if verbose:
1019                         logger.debug(r)
1020                 return 0
1021
1022         except urllib2.URLError,err:
1023                 logger.info('Could not open http connection', err)
1024                 return "bbpse error"
1025
1026 ### rebooting x10toggle based systems addressed by port
1027 # Marc E. Fiuczynski - May 31 2005
1028 # tested on 4-ports models known as PSE505-FR
1029 # uses ssh and password to login to an account
1030 # that will cause the system to be powercycled.
1031
1032 def x10toggle_reboot(ip, username, password, port, dryrun):
1033         global verbose
1034
1035         ssh = None
1036         try:
1037                 ssh = pyssh.Ssh(username, ip)
1038                 ssh.open()
1039
1040                 # Login
1041                 telnet_answer(ssh, "password:", password)
1042
1043                 if not dryrun:
1044                         # Reboot
1045                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
1046
1047                 # Close
1048                 output = ssh.close()
1049                 if verbose:
1050                         logger.debug(output)
1051                 return 0
1052
1053         except Exception, err:
1054                 if verbose:
1055                         logger.debug(err)
1056                 if ssh:
1057                         output = ssh.close()
1058                         if verbose:
1059                                 logger.debug(output)
1060                 return errno.ETIMEDOUT
1061
1062 ### rebooting Dell systems via RAC card
1063 # Marc E. Fiuczynski - June 01 2005
1064 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1065 #
1066
1067 def runcmd(command, args, username, password, timeout = None):
1068
1069         result = [None]
1070         result_ready = threading.Condition()
1071
1072         def set_result(x):
1073
1074                 result_ready.acquire()
1075                 try:
1076                         result[0] = x
1077                 finally:
1078                         result_ready.notify()
1079                         result_ready.release()
1080
1081         def do_command(command, username, password):
1082
1083                 try:
1084                         # Popen4 is a popen-type class that combines stdout and stderr
1085                         p = popen2.Popen4(command)
1086
1087                         # read all output data
1088                         p.tochild.write("%s\n" % username)
1089                         p.tochild.write("%s\n" % password)
1090                         p.tochild.close()
1091                         data = p.fromchild.read()
1092
1093                         while True:
1094                                 # might get interrupted by a signal in poll() or waitpid()
1095                                 try:
1096                                         retval = p.wait()
1097                                         set_result((retval, data))
1098                                         break
1099                                 except OSError, ex:
1100                                         if ex.errno == errno.EINTR:
1101                                                 continue
1102                                         raise ex
1103                 except Exception, ex:
1104                         set_result(ex)
1105
1106         if args:
1107                 command = " ".join([command] + args)
1108
1109         worker = threading.Thread(target = do_command, args = (command, username, password, ))
1110         worker.setDaemon(True)
1111         result_ready.acquire()
1112         worker.start()
1113         result_ready.wait(timeout)
1114         try:
1115                 if result == [None]:
1116                         raise Exception, "command timed-out: '%s'" % command
1117         finally:
1118                 result_ready.release()
1119         result = result[0]
1120
1121         if isinstance(result, Exception):
1122                 raise result
1123         else:
1124                 (retval, data) = result
1125                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1126                         return data
1127                 else:
1128                         out = "system command ('%s') " % command
1129                         if os.WIFEXITED(retval):
1130                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1131                         else:
1132                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1133                         if data:
1134                                 out += "; output follows:\n" + data
1135                         raise Exception, out
1136
1137 def racadm_reboot(host, username, password, port, dryrun):
1138         global verbose
1139
1140         ip = socket.gethostbyname(host)
1141         try:
1142                 cmd = "/usr/sbin/racadm"
1143                 os.stat(cmd)
1144                 if not dryrun:
1145                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1146                                 username, password)
1147                 else:
1148                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1149                                 username, password)
1150
1151                 print "RUNCMD: %s" % output
1152                 if verbose:
1153                         logger.debug(output)
1154                 return 0
1155
1156         except Exception, err:
1157                 logger.debug("runcmd raised exception %s" % err)
1158                 if verbose:
1159                         logger.debug(err)
1160                 return -1
1161
1162 def pcu_name(pcu):
1163         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1164                 return pcu['hostname']
1165         elif pcu['ip'] is not None and pcu['ip'] is not "":
1166                 return pcu['ip']
1167         else:
1168                 return None
1169
1170 import database
1171 fb =database.dbLoad("findbadpcus")
1172
1173 def get_pcu_values(pcu_id):
1174         # TODO: obviously, this shouldn't be loaded each time...
1175
1176         try:
1177                 values = fb['nodes']["id_%s" % pcu_id]['values']
1178         except:
1179                 values = None
1180
1181         return values
1182
1183 def reboot(nodename):
1184         return reboot_policy(nodename, True, False)
1185
1186 def reboot_policy(nodename, continue_probe, dryrun):
1187         global verbose
1188
1189         pcu = plc.getpcu(nodename)
1190         if not pcu:
1191                 logger.debug("no pcu for %s" % hostname)
1192                 print "no pcu for %s" % hostname
1193                 return False # "%s has no pcu" % nodename
1194
1195         values = get_pcu_values(pcu['pcu_id'])
1196         if values == None:
1197                 logger.debug("No values for pcu probe %s" % hostname)
1198                 print "No values for pcu probe %s" % hostname
1199                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1200
1201         # Try the PCU first
1202         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1203
1204         ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1205
1206         if ret != 0:
1207                 print ret
1208                 return False
1209         else:
1210                 print "return true"
1211                 return True
1212
1213 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1214         rb_ret = ""
1215
1216         try:
1217                 # DataProbe iPal (many sites)
1218                 if  continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1219                         ipal = IPAL(values, verbose, ['23', '80', '9100'])
1220                         rb_ret = ipal.reboot(values[nodename], dryrun)
1221
1222                 # APC Masterswitch (Berkeley)
1223                 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1224                                                                   values['model'].find("Masterswitch") >= 0 ):
1225                         print values
1226
1227                         # TODO: make a more robust version of APC
1228                         if values['pcu_id'] in [1163,1055,1111,1231,1113,1127,1128,1148]:
1229                                 apc = APCEurope(values, verbose, ['22', '23'])
1230                                 rb_ret = apc.reboot(values[nodename], dryrun)
1231
1232                         elif values['pcu_id'] in [1110,86]:
1233                                 apc = APCBrazil(values, verbose, ['22', '23'])
1234                                 rb_ret = apc.reboot(values[nodename], dryrun)
1235
1236                         elif values['pcu_id'] in [1221,1225]:
1237                                 apc = APCBerlin(values, verbose, ['22', '23'])
1238                                 rb_ret = apc.reboot(values[nodename], dryrun)
1239
1240                         elif values['pcu_id'] in [1173,1221,1220]:
1241                                 apc = APCFolsom(values, verbose, ['22', '23'])
1242                                 rb_ret = apc.reboot(values[nodename], dryrun)
1243
1244                         else:
1245                                 apc = APCMaster(values, verbose, ['22', '23'])
1246                                 rb_ret = apc.reboot(values[nodename], dryrun)
1247
1248                 # BayTech DS4-RPC
1249                 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1250                         if values['pcu_id'] in [1237,1052,1209,1002,1008,1041,1013,1022]:
1251                                 # These  require a 'ctrl-c' to be sent...
1252                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1253                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1254
1255                         elif values['pcu_id'] in [93]:
1256                                 baytech = BayTechAU(values, verbose, ['22', '23'])
1257                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1258
1259                         elif values['pcu_id'] in [1057]:
1260                                 # These  require a 'ctrl-c' to be sent...
1261                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1262                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1263
1264                         elif values['pcu_id'] in [1012]:
1265                                 # This pcu sometimes doesn't present the 'Username' prompt,
1266                                 # unless you immediately try again...
1267                                 try:
1268                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1269                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1270                                 except:
1271                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1272                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1273                         else:
1274                                 baytech = BayTech(values, verbose, ['22', '23'])
1275                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1276
1277                 # iLO
1278                 elif continue_probe and values['model'].find("ilo") >= 0:
1279                         try:
1280                                 hpilo = HPiLO(values, verbose, ['22'])
1281                                 rb_ret = hpilo.reboot(0, dryrun)
1282                                 if rb_ret != 0:
1283                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1284                                         rb_ret = hpilo.reboot(0, dryrun)
1285                         except:
1286                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1287                                 rb_ret = hpilo.reboot(0, dryrun)
1288
1289                 # DRAC ssh
1290                 elif continue_probe and values['model'].find("DRAC") >= 0:
1291                         # TODO: I don't think DRACRacAdm will throw an exception for the
1292                         # default method to catch...
1293                         try:
1294                                 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1295                                 rb_ret = drac.reboot(0, dryrun)
1296                         except:
1297                                 drac = DRAC(values, verbose, ['22'])
1298                                 rb_ret = drac.reboot(0, dryrun)
1299
1300                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1301                                 wti = WTIIPS4(values, verbose, ['23'])
1302                                 rb_ret = wti.reboot(values[nodename], dryrun)
1303
1304                 elif continue_probe and values['model'].find("AMT") >= 0:
1305                                 amt = IntelAMT(values, verbose, ['16992'])
1306                                 rb_ret = amt.reboot(values[nodename], dryrun)
1307
1308                 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1309                 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1310                         # TODO: allow a different port than http 80.
1311                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1312                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1313                         elif values['pcu_id'] in [1003]:
1314                                 # OLD EPOWER
1315                                 print "OLD EPOWER"
1316                                 eps = ePowerSwitch(values, verbose, ['80'])
1317                         else:
1318                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1319
1320                         rb_ret = eps.reboot(values[nodename], dryrun)
1321                 elif continue_probe and values['pcu_id'] in [1122]:
1322                         custom = CustomPCU(values, verbose, ['80', '443'])
1323                         custom.reboot(values[nodename], dryrun)
1324
1325                 elif continue_probe:
1326                         rb_ret = "Unsupported_PCU"
1327
1328                 elif continue_probe == False:
1329                         if 'portstatus' in values:
1330                                 rb_ret = "NetDown"
1331                         else:
1332                                 rb_ret = "Not_Run"
1333                 else:
1334                         rb_ret = -1
1335
1336         except ExceptionPort, err:
1337                 rb_ret = str(err)
1338
1339         return rb_ret
1340         # ????
1341         #elif continue_probe and values['protocol'] == "racadm" and \
1342         #               values['model'] == "RAC":
1343         #       rb_ret = racadm_reboot(pcu_name(values),
1344         #                                                                 values['username'],
1345         #                                                                 values['password'],
1346         #                                                                 pcu[nodename],
1347         #                                                                 dryrun)
1348
1349 def main():
1350         logger.setLevel(logging.DEBUG)
1351         ch = logging.StreamHandler()
1352         ch.setLevel(logging.DEBUG)
1353         formatter = logging.Formatter('LOGGER - %(message)s')
1354         ch.setFormatter(formatter)
1355         logger.addHandler(ch)
1356
1357         try:
1358                 if "test" in sys.argv:
1359                         dryrun = True
1360                 else:
1361                         dryrun = False
1362
1363                 for node in sys.argv[1:]:
1364                         if node == "test": continue
1365
1366                         print "Rebooting %s" % node
1367                         if reboot_policy(node, True, dryrun):
1368                                 print "success"
1369                         else:
1370                                 print "failed"
1371         except Exception, err:
1372                 import traceback; traceback.print_exc()
1373                 print err
1374
1375 if __name__ == '__main__':
1376         import plc
1377         logger = logging.getLogger("monitor")
1378         main()