reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import urllib
  12 import threading, popen2
  13 import array, struct
  14 import plc
  15 import base64
  16 from subprocess import PIPE, Popen
  17 import ssh.pxssh as pxssh
  18 import ssh.pexpect as pexpect
  19 import socket
  20 import moncommands
  21
  22 # Use our versions of telnetlib and pyssh
  23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  24 import telnetlib
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  26 import pyssh
  27
  28 # Timeouts in seconds
  29 TELNET_TIMEOUT = 45
  30
  31 # Event class ID from pcu events
  32 #NODE_POWER_CONTROL = 3
  33
  34 # Monitor user ID
  35 #MONITOR_USER_ID = 11142
  36
  37 import logging
  38 logger = logging.getLogger("monitor")
  39 verbose = 1
  40 #dryrun = 0;
  41
  42 class ExceptionNoTransport(Exception): pass
  43 class ExceptionNotFound(Exception): pass
  44 class ExceptionPassword(Exception): pass
  45 class ExceptionTimeout(Exception): pass
  46 class ExceptionPrompt(Exception): pass
  47 class ExceptionSequence(Exception): pass
  48 class ExceptionReset(Exception): pass
  49 class ExceptionPort(Exception): pass
  50 class ExceptionUsername(Exception): pass
  51
  52 def telnet_answer(telnet, expected, buffer):
  53         global verbose
  54
  55         output = telnet.read_until(expected, TELNET_TIMEOUT)
  56         #if verbose:
  57         #       logger.debug(output)
  58         if output.find(expected) == -1:
  59                 raise ExceptionNotFound, "'%s' not found" % expected
  60         else:
  61                 telnet.write(buffer + "\r\n")
  62
  63
  64 # PCU has model, host, preferred-port, user, passwd,
  65
  66 # This is an object derived directly form the PLCAPI DB fields
  67 class PCU(object):
  68         def __init__(self, plc_pcu_dict):
  69                 for field in ['username', 'password', 'site_id',
  70                                                 'hostname', 'ip',
  71                                                 'pcu_id', 'model',
  72                                                 'node_ids', 'ports', ]:
  73                         if field in plc_pcu_dict:
  74                                 self.__setattr__(field, plc_pcu_dict[field])
  75                         else:
  76                                 raise Exception("No such field %s in PCU object" % field)
  77
  78 # These are the convenience functions build around the PCU object.
  79 class PCUModel(PCU):
  80         def __init__(self, plc_pcu_dict):
  81                 PCU.__init__(self, plc_pcu_dict)
  82                 self.host = self.pcu_name()
  83
  84         def pcu_name(self):
  85                 if self.hostname is not None and self.hostname is not "":
  86                         return self.hostname
  87                 elif self.ip is not None and self.ip is not "":
  88                         return self.ip
  89                 else:
  90                         return None
  91
  92         def nodeidToPort(self, node_id):
  93                 if node_id in self.node_ids:
  94                         for i in range(0, len(self.node_ids)):
  95                                 if node_id == self.node_ids[i]:
  96                                         return self.ports[i]
  97
  98                 raise Exception("No such Node ID: %d" % node_id)
  99
 100 # This class captures the observed pcu records from FindBadPCUs.py
 101 class PCURecord:
 102         def __init__(self, pcu_record_dict):
 103                 for field in ['nodenames', 'portstatus',
 104                                                 'dnsmatch',
 105                                                 'complete_entry', ]:
 106                         if field in pcu_record_dict:
 107                                 if field == "reboot":
 108                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 109                                 else:
 110                                         self.__setattr__(field, pcu_record_dict[field])
 111                         else:
 112                                 raise Exception("No such field %s in pcu record dict" % field)
 113
 114 class Transport:
 115         TELNET = 1
 116         SSH    = 2
 117         HTTP   = 3
 118         IPAL   = 4
 119
 120         TELNET_TIMEOUT = 60
 121
 122         def __init__(self, type, verbose):
 123                 self.type = type
 124                 self.verbose = verbose
 125                 self.transport = None
 126
 127         def open(self, host, username=None, password=None, prompt="User Name"):
 128                 transport = None
 129
 130                 if self.type == self.TELNET:
 131                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 132                         transport.set_debuglevel(self.verbose)
 133                         if username is not None:
 134                                 self.transport = transport
 135                                 self.ifThenSend(prompt, username, ExceptionUsername)
 136
 137                 elif self.type == self.SSH:
 138                         if username is not None:
 139                                 transport = pyssh.Ssh(username, host)
 140                                 transport.set_debuglevel(self.verbose)
 141                                 transport.open()
 142                                 # TODO: have an ssh set_debuglevel() also...
 143                         else:
 144                                 raise Exception("Username cannot be None for ssh transport.")
 145                 elif self.type == self.HTTP:
 146                         self.url = "http://%s:%d/" % (host,80)
 147                         uri = "%s:%d" % (host,80)
 148
 149                         # create authinfo
 150                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 151                         authinfo.add_password (None, uri, username, password)
 152                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 153
 154                         transport = urllib2.build_opener(authhandler)
 155
 156                 else:
 157                         raise Exception("Unknown transport type: %s" % self.type)
 158
 159                 self.transport = transport
 160                 return True
 161
 162         def close(self):
 163                 if self.type == self.TELNET:
 164                         self.transport.close()
 165                 elif self.type == self.SSH:
 166                         self.transport.close()
 167                 elif self.type == self.HTTP:
 168                         pass
 169                 else:
 170                         raise Exception("Unknown transport type %s" % self.type)
 171                 self.transport = None
 172
 173         def sendHTTP(self, resource, data):
 174                 if self.verbose:
 175                         print "POSTing '%s' to %s" % (data,self.url + resource)
 176
 177                 try:
 178                         f = self.transport.open(self.url + resource ,data)
 179                         r = f.read()
 180                         if self.verbose:
 181                                 print r
 182
 183                 except urllib2.URLError,err:
 184                         logger.info('Could not open http connection', err)
 185                         return "http transport error"
 186
 187                 return 0
 188
 189         def sendPassword(self, password, prompt=None):
 190                 if self.type == self.TELNET:
 191                         if prompt == None:
 192                                 self.ifThenSend("Password", password, ExceptionPassword)
 193                         else:
 194                                 self.ifThenSend(prompt, password, ExceptionPassword)
 195                 elif self.type == self.SSH:
 196                         self.ifThenSend("password:", password, ExceptionPassword)
 197                 elif self.type == self.HTTP:
 198                         pass
 199                 else:
 200                         raise Exception("Unknown transport type: %s" % self.type)
 201
 202         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 203
 204                 if self.transport != None:
 205                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 206                         if output.find(expected) == -1:
 207                                 raise ErrorClass, "'%s' not found" % expected
 208                         else:
 209                                 self.transport.write(buffer + "\r\n")
 210                 else:
 211                         raise ExceptionNoTransport("transport object is type None")
 212
 213         def ifElse(self, expected, ErrorClass):
 214                 try:
 215                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 216                 except:
 217                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 218
 219
 220 class PCUControl(Transport,PCUModel,PCURecord):
 221         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 222                 PCUModel.__init__(self, plc_pcu_record)
 223                 PCURecord.__init__(self, plc_pcu_record)
 224                 type = None
 225                 if self.portstatus:
 226                         if '22' in supported_ports and self.portstatus['22'] == "open":
 227                                 type = Transport.SSH
 228                         elif '23' in supported_ports and self.portstatus['23'] == "open":
 229                                 type = Transport.TELNET
 230                         elif '80' in supported_ports and self.portstatus['80'] == "open":
 231                                 type = Transport.HTTP
 232                         elif '443' in supported_ports and self.portstatus['443'] == "open":
 233                                 type = Transport.HTTP
 234                         elif '5869' in supported_ports and self.portstatus['5869'] == "open":
 235                                 # For DRAC cards. Racadm opens this port.
 236                                 type = Transport.HTTP
 237                         elif '9100' in supported_ports and self.portstatus['9100'] == "open":
 238                                 type = Transport.IPAL
 239                         elif '16992' in supported_ports and self.portstatus['16992'] == "open":
 240                                 type = Transport.HTTP
 241                         else:
 242                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 243                 else:
 244                         raise Exception("No Portstatus: No transport because no open ports")
 245                 Transport.__init__(self, type, verbose)
 246
 247         def run(self, node_port, dryrun):
 248                 """ This function is to be defined by the specific PCU instance.  """
 249                 pass
 250
 251         def reboot(self, node_port, dryrun):
 252                 try:
 253                         return self.run(node_port, dryrun)
 254                 except ExceptionNotFound, err:
 255                         return "error: " + str(err)
 256                 except ExceptionPassword, err:
 257                         return "password exception: " + str(err)
 258                 except ExceptionTimeout, err:
 259                         return "timeout exception: " + str(err)
 260                 except ExceptionUsername, err:
 261                         return "exception: no username prompt: " + str(err)
 262                 except ExceptionSequence, err:
 263                         return "sequence error: " + str(err)
 264                 except ExceptionPrompt, err:
 265                         return "prompt exception: " + str(err)
 266                 except ExceptionPort, err:
 267                         return "no ports exception: " + str(err)
 268                 except socket.error, err:
 269                         return "socket error: timeout: " + str(err)
 270                 except EOFError, err:
 271                         if self.verbose:
 272                                 logger.debug("reboot: EOF")
 273                                 logger.debug(err)
 274                         self.transport.close()
 275                         import traceback
 276                         traceback.print_exc()
 277                         return "EOF connection reset" + str(err)
 278                 except:
 279                         from nodecommon import email_exception
 280                         email_exception()
 281                         raise Exception('unknown')
 282
 283 class IPAL(PCUControl):
 284         """
 285                 This now uses a proprietary format for communicating with the PCU.  I
 286                 prefer it to Telnet, and Web access, since it's much lighter weight
 287                 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
 288         """
 289
 290         def format_msg(self, data, cmd):
 291                 esc = chr(int('1b',16))
 292                 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
 293
 294         def recv_noblock(self, s, count):
 295                 import errno
 296
 297                 try:
 298                         # TODO: make sleep backoff, before stopping.
 299                         time.sleep(4)
 300                         ret = s.recv(count, socket.MSG_DONTWAIT)
 301                 except socket.error, e:
 302                         if e[0] == errno.EAGAIN:
 303                                 raise Exception(e[1])
 304                         else:
 305                                 # TODO: not other exceptions.
 306                                 raise Exception(e)
 307                 return ret
 308
 309         def run(self, node_port, dryrun):
 310                 import errno
 311
 312                 power_on = False
 313
 314                 print "open socket"
 315                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 316                 try:
 317                         print "connect"
 318                         s.connect((self.host, 9100))
 319                 except socket.error, e:
 320                         s.close()
 321                         if e[0] == errno.ECONNREFUSED:
 322                                 # cannot connect to remote host
 323                                 raise Exception(e[1])
 324                         else:
 325                                 # TODO: what other conditions are there?
 326                                 raise Exception(e)
 327
 328                 # get current status
 329                 print "Checking status"
 330                 s.send(self.format_msg("", 'O'))
 331                 ret = self.recv_noblock(s, 8)
 332                 print "Current status is '%s'" % ret
 333
 334                 if ret == '':
 335                         raise Exception("Status returned 'another session already open' on %s %s : %s" % (self.host, node_port, ret))
 336
 337
 338                 if node_port < len(ret):
 339                         status = ret[node_port]
 340                         if status == '1':
 341                                 # up
 342                                 power_on = True
 343                         elif status == '0':
 344                                 # down
 345                                 power_on = False
 346                         elif status == '6':
 347                                 raise ExceptionPort("IPAL reported 'Cable Error' on %s socket %s : %s" % (self.host, node_port, ret))
 348                         else:
 349                                 raise Exception("Unknown status for PCU %s socket %s : %s" % (self.host, node_port, ret))
 350                 else:
 351                         raise Exception("Mismatch between configured port and PCU %s status: %s %s" % (self.host, node_port, ret))
 352
 353
 354                 if not dryrun:
 355                         if power_on:
 356                                 print "Pulsing %s" % node_port
 357                                 s.send(self.format_msg("%s" % node_port, 'P'))
 358                         else:
 359                                 # NOTE: turn power on ; do not pulse the port.
 360                                 print "Power was off, so turning on ..."
 361                                 s.send(self.format_msg("%s" % node_port, 'E'))
 362                                 #s.send(self.format_msg("%s" % node_port, 'P'))
 363
 364                         print "Receiving response."
 365                         ret = self.recv_noblock(s, 8)
 366                         print "Current status is '%s'" % ret
 367
 368                         if node_port < len(ret):
 369                                 status = ret[node_port]
 370                                 if status == '1':
 371                                         # up
 372                                         power_on = True
 373                                 elif status == '0':
 374                                         # down
 375                                         power_on = False
 376                                 elif status == '6':
 377                                         raise ExceptionPort("IPAL reported 'Cable Error' on %s socket %s : %s" % (self.host, node_port, ret))
 378                                 else:
 379                                         raise Exception("Unknown status for PCU %s socket %s : %s" % (self.host, node_port, ret))
 380                         else:
 381                                 raise Exception("Mismatch between configured port and PCU %s status: %s %s" % (self.host, node_port, ret))
 382
 383                         if power_on:
 384                                 return 0
 385                         else:
 386                                 return "Failed Power On"
 387
 388                 s.close()
 389                 return 0
 390
 391 # TELNET version of protocol...
 392 #               #self.open(self.host)
 393 #               ## XXX Some iPals require you to hit Enter a few times first
 394 #               #self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 395 #               # Login
 396 #               self.ifThenSend("Password >", self.password, ExceptionPassword)
 397 #               self.transport.write("\r\n\r\n")
 398 #               if not dryrun: # P# - Pulse relay
 399 #                       print "node_port %s" % node_port
 400 #                       self.ifThenSend("Enter >",
 401 #                                                       "P7", # % node_port,
 402 #                                                       ExceptionNotFound)
 403 #                       print "send newlines"
 404 #                       self.transport.write("\r\n\r\n")
 405 #                       print "after new lines"
 406 #               # Get the next prompt
 407 #               print "wait for enter"
 408 #               self.ifElse("Enter >", ExceptionTimeout)
 409 #               print "closing "
 410 #               self.close()
 411 #               return 0
 412
 413 class APCEurope(PCUControl):
 414         def run(self, node_port, dryrun):
 415                 self.open(self.host, self.username)
 416                 self.sendPassword(self.password)
 417
 418                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 419                 self.ifThenSend("\r\n> ", "2")
 420                 self.ifThenSend("\r\n> ", str(node_port))
 421                 # 3- Immediate Reboot
 422                 self.ifThenSend("\r\n> ", "3")
 423
 424                 if not dryrun:
 425                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 426                                                         "YES\r\n",
 427                                                         ExceptionSequence)
 428                 else:
 429                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 430                                                         "", ExceptionSequence)
 431                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 432
 433                 self.close()
 434                 return 0
 435
 436 class APCBrazil(PCUControl):
 437         def run(self, node_port, dryrun):
 438                 self.open(self.host, self.username)
 439                 self.sendPassword(self.password)
 440
 441                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 442                 self.ifThenSend("\r\n> ", str(node_port))
 443                 # 4- Immediate Reboot
 444                 self.ifThenSend("\r\n> ", "4")
 445
 446                 if not dryrun:
 447                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 448                                                         "YES\r\n",
 449                                                         ExceptionSequence)
 450                 else:
 451                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 452                                                         "", ExceptionSequence)
 453                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 454
 455                 self.close()
 456                 return 0
 457
 458 class APCBerlin(PCUControl):
 459         def run(self, node_port, dryrun):
 460                 self.open(self.host, self.username)
 461                 self.sendPassword(self.password)
 462
 463                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 464                 self.ifThenSend("\r\n> ", "2")
 465                 self.ifThenSend("\r\n> ", "1")
 466                 self.ifThenSend("\r\n> ", str(node_port))
 467                 # 3- Immediate Reboot
 468                 self.ifThenSend("\r\n> ", "3")
 469
 470                 if not dryrun:
 471                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 472                                                         "YES\r\n",
 473                                                         ExceptionSequence)
 474                 else:
 475                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 476                                                         "", ExceptionSequence)
 477                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 478
 479                 self.close()
 480                 return 0
 481
 482 class APCFolsom(PCUControl):
 483         def run(self, node_port, dryrun):
 484                 self.open(self.host, self.username)
 485                 self.sendPassword(self.password)
 486
 487                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 488                 self.ifThenSend("\r\n> ", "2")
 489                 self.ifThenSend("\r\n> ", "1")
 490                 self.ifThenSend("\r\n> ", str(node_port))
 491                 self.ifThenSend("\r\n> ", "1")
 492
 493                 # 3- Immediate Reboot
 494                 self.ifThenSend("\r\n> ", "3")
 495
 496                 if not dryrun:
 497                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 498                                                         "YES\r\n",
 499                                                         ExceptionSequence)
 500                 else:
 501                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 502                                                         "", ExceptionSequence)
 503                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 504
 505                 self.close()
 506                 return 0
 507
 508 class APCMaster(PCUControl):
 509         def run(self, node_port, dryrun):
 510                 print "Rebooting %s" % self.host
 511                 self.open(self.host, self.username)
 512                 self.sendPassword(self.password)
 513
 514                 # 1- Device Manager
 515                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 516                 # 3- Outlet Control/Config
 517                 self.ifThenSend("\r\n> ", "3")
 518                 # n- Outlet n
 519                 self.ifThenSend("\r\n> ", str(node_port))
 520                 # 1- Control Outlet
 521                 self.ifThenSend("\r\n> ", "1")
 522                 # 3- Immediate Reboot
 523                 self.ifThenSend("\r\n> ", "3")
 524
 525                 if not dryrun:
 526                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 527                                                         "YES\r\n",
 528                                                         ExceptionSequence)
 529                 else:
 530                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 531                                                         "", ExceptionSequence)
 532                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 533
 534                 self.close()
 535                 return 0
 536
 537 class APC(PCUControl):
 538         def __init__(self, plc_pcu_record, verbose):
 539                 PCUControl.__init__(self, plc_pcu_record, verbose)
 540
 541                 self.master = APCMaster(plc_pcu_record, verbose)
 542                 self.folsom = APCFolsom(plc_pcu_record, verbose)
 543                 self.europe = APCEurope(plc_pcu_record, verbose)
 544
 545         def run(self, node_port, dryrun):
 546                 try_again = True
 547                 sleep_time = 1
 548
 549                 for pcu in [self.master, self.europe, self.folsom]:
 550                         if try_again:
 551                                 try:
 552                                         print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
 553                                         try_again = False
 554                                         print "sleeping 5"
 555                                         time.sleep(sleep_time)
 556                                         ret = pcu.reboot(node_port, dryrun)
 557                                 except ExceptionSequence, err:
 558                                         del pcu
 559                                         sleep_time = 130
 560                                         try_again = True
 561
 562                 if try_again:
 563                         return "Unknown reboot sequence for APC PCU"
 564                 else:
 565                         return ret
 566
 567 class IntelAMT(PCUControl):
 568         def run(self, node_port, dryrun):
 569
 570                 cmd = moncommands.CMD()
 571                 #[cmd_str = "IntelAMTSDK/Samples/RemoteControl/remoteControl"
 572                 cmd_str = "cmdamt/remoteControl"
 573
 574                 if dryrun:
 575                         # NOTE: -p checks the power state of the host.
 576                         # TODO: parse the output to find out if it's ok or not.
 577                         cmd_str += " -p http://%s:16992/RemoteControlService  -user admin -pass '%s' " % (self.host, self.password )
 578                 else:
 579                         cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
 580
 581                 print cmd_str
 582                 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
 583
 584 class DRACRacAdm(PCUControl):
 585         def run(self, node_port, dryrun):
 586
 587                 print "trying racadm_reboot..."
 588                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 589
 590                 return 0
 591
 592 class DRAC(PCUControl):
 593         def run(self, node_port, dryrun):
 594                 self.open(self.host, self.username)
 595                 self.sendPassword(self.password)
 596
 597                 print "logging in..."
 598                 self.transport.write("\r\n")
 599                 # Testing Reboot ?
 600                 if dryrun:
 601                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 602                 else:
 603                         # Reset this machine
 604                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 605
 606                 self.ifThenSend("[%s]#" % self.username, "exit")
 607
 608                 self.close()
 609                 return 0
 610
 611 class HPiLO(PCUControl):
 612         def run(self, node_port, dryrun):
 613                 self.open(self.host, self.username)
 614                 self.sendPassword(self.password)
 615
 616                 # </>hpiLO->
 617                 self.ifThenSend("</>hpiLO->", "cd system1")
 618
 619                 # Reboot Outlet  N        (Y/N)?
 620                 if dryrun:
 621                         self.ifThenSend("</system1>hpiLO->", "POWER")
 622                 else:
 623                         # Reset this machine
 624                         self.ifThenSend("</system1>hpiLO->", "reset")
 625
 626                 self.ifThenSend("</system1>hpiLO->", "exit")
 627
 628                 self.close()
 629                 return 0
 630
 631
 632 class HPiLOHttps(PCUControl):
 633         def run(self, node_port, dryrun):
 634
 635                 locfg = moncommands.CMD()
 636                 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 637                                         self.host, "iloxml/Get_Network.xml",
 638                                         self.username, self.password)
 639                 sout, serr = locfg.run_noexcept(cmd)
 640
 641                 if sout.strip() != "":
 642                         print "sout: %s" % sout.strip()
 643                         return sout.strip()
 644
 645                 if not dryrun:
 646                         locfg = moncommands.CMD()
 647                         cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 648                                                 self.host, "iloxml/Reset_Server.xml",
 649                                                 self.username, self.password)
 650                         sout, serr = locfg.run_noexcept(cmd)
 651
 652                         if sout.strip() != "":
 653                                 print "sout: %s" % sout.strip()
 654                                 #return sout.strip()
 655                 return 0
 656
 657 class BayTechAU(PCUControl):
 658         def run(self, node_port, dryrun):
 659                 self.open(self.host, self.username, None, "Enter user name:")
 660                 self.sendPassword(self.password, "Enter Password:")
 661
 662                 #self.ifThenSend("RPC-16>", "Status")
 663                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 664
 665                 # Reboot Outlet  N        (Y/N)?
 666                 if dryrun:
 667                         self.ifThenSend("(Y/N)?", "N")
 668                 else:
 669                         self.ifThenSend("(Y/N)?", "Y")
 670                 self.ifThenSend("RPC3-NC>", "")
 671
 672                 self.close()
 673                 return 0
 674
 675 class BayTechGeorgeTown(PCUControl):
 676         def run(self, node_port, dryrun):
 677                 # this initial open/close is to prevent things from raising an
 678                 # exception.  the pcu always is weird during the first connection, and
 679                 # even if it's not, what does it matter to open a second connection
 680                 # right away?
 681                 self.open(self.host, self.username, None, "Enter user name:")
 682                 self.close()
 683                 time.sleep(1)
 684                 self.open(self.host, self.username, None, "Enter user name:")
 685                 self.sendPassword(self.password, "Enter Password:")
 686
 687                 #self.ifThenSend("RPC-16>", "Status")
 688
 689                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 690
 691                 # Reboot Outlet  N        (Y/N)?
 692                 if dryrun:
 693                         self.ifThenSend("(Y/N)?", "N")
 694                 else:
 695                         self.ifThenSend("(Y/N)?", "Y")
 696                 self.ifThenSend("RPC-16>", "")
 697
 698                 self.close()
 699                 return 0
 700
 701 class BayTechCtrlCUnibe(PCUControl):
 702         """
 703                 For some reason, these units let you log in fine, but they hang
 704                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 705                 why.
 706         """
 707         def run(self, node_port, dryrun):
 708                 print "BayTechCtrlC %s" % self.host
 709
 710                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 711                 s = pxssh.pxssh()
 712                 if not s.login(self.host, self.username, self.password, ssh_options):
 713                         raise ExceptionPassword("Invalid Password")
 714                 # Otherwise, the login succeeded.
 715
 716                 # Send a ctrl-c to the remote process.
 717                 print "sending ctrl-c"
 718                 s.send(chr(3))
 719
 720                 # Control Outlets  (5 ,1).........5
 721                 try:
 722                         #index = s.expect("Enter Request")
 723                         index = s.expect(["Enter Request :"])
 724
 725                         if index == 0:
 726                                 print "3"
 727                                 s.send("3\r\n")
 728                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 729                                 if index == 1:
 730                                         s.send(self.username + "\r\n")
 731                                         index = s.expect(["DS-RPC>"])
 732
 733                                 if index == 0:
 734                                         print "Reboot %d" % node_port
 735                                         s.send("Reboot %d\r\n" % node_port)
 736
 737                                         time.sleep(5)
 738                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 739                                         if index == 0:
 740                                                 if dryrun:
 741                                                         print "sending N"
 742                                                         s.send("N\r\n")
 743                                                 else:
 744                                                         print "sending Y"
 745                                                         s.send("Y\r\n")
 746                                         elif index == 1:
 747                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 748                                         elif index == 2:
 749                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 750
 751                                 time.sleep(5)
 752                                 index = s.expect(["DS-RPC>"])
 753                                 #print "got prompt back"
 754
 755                         s.close()
 756
 757                 except pexpect.EOF:
 758                         raise ExceptionPrompt("EOF before expected Prompt")
 759                 except pexpect.TIMEOUT:
 760                         raise ExceptionPrompt("Timeout before expected Prompt")
 761
 762                 return 0
 763
 764 class BayTechCtrlC(PCUControl):
 765         """
 766                 For some reason, these units let you log in fine, but they hang
 767                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 768                 why.
 769         """
 770         def run(self, node_port, dryrun):
 771                 print "BayTechCtrlC %s" % self.host
 772
 773                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 774                 s = pxssh.pxssh()
 775                 if not s.login(self.host, self.username, self.password, ssh_options):
 776                         raise ExceptionPassword("Invalid Password")
 777                 # Otherwise, the login succeeded.
 778
 779                 # Send a ctrl-c to the remote process.
 780                 print "SENDING ctrl-c"
 781                 s.send(chr(3))
 782
 783                 # Control Outlets  (5 ,1).........5
 784                 try:
 785                         print "EXPECTING: ", "Enter Request :"
 786                         index = s.expect(["Enter Request :"])
 787
 788                         if index == 0:
 789                                 print "SENDING: 5"
 790                                 s.send("5\r\n")
 791                                 print "EXPECTING: ", "DS-RPC>"
 792                                 index = s.expect(["DS-RPC>", "Enter user name:", "Port in use."])
 793                                 if index == 1:
 794                                         print "sending username"
 795                                         s.send(self.username + "\r\n")
 796                                         index = s.expect(["DS-RPC>"])
 797                                 elif index == 2:
 798                                         raise ExceptionPrompt("PCU Reported 'Port in use.'")
 799
 800                                 if index == 0:
 801                                         print "SENDING: Reboot %d" % node_port
 802                                         s.send("Reboot %d\r\n" % node_port)
 803
 804                                         print "SLEEPING: 5"
 805                                         time.sleep(5)
 806                                         print "EXPECTING: ", "Y/N?"
 807                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 808                                         if index == 0:
 809                                                 if dryrun:
 810                                                         print "sending N"
 811                                                         s.send("N\r\n")
 812                                                 else:
 813                                                         print "SENDING: Y"
 814                                                         s.send("Y\r\n")
 815                                         elif index == 1:
 816                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 817                                         elif index == 2:
 818                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 819
 820                                 # NOTE: for some reason, the script times out with the
 821                                 # following line.  In manual tests, it works correctly, but
 822                                 # with automated tests, evidently it fails.
 823                                 print "SLEEPING: 5"
 824                                 time.sleep(5)
 825                                 #print "TOTAL--", s.allstr, "--EOT"
 826                                 index = s.expect(["DS-RPC>"])
 827                                 print "got prompt back"
 828
 829                         s.close()
 830
 831                 except pexpect.EOF:
 832                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 833                 except pexpect.TIMEOUT:
 834                         raise ExceptionPrompt("Timeout before Prompt")
 835
 836                 return 0
 837
 838 class BayTech(PCUControl):
 839         def run(self, node_port, dryrun):
 840                 self.open(self.host, self.username)
 841                 self.sendPassword(self.password)
 842
 843                 # Control Outlets  (5 ,1).........5
 844                 self.ifThenSend("Enter Request :", "5")
 845
 846                 # Reboot N
 847                 try:
 848                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 849                 except ExceptionNotFound, msg:
 850                         # one machine is configured to ask for a username,
 851                         # even after login...
 852                         print "msg: %s" % msg
 853                         self.transport.write(self.username + "\r\n")
 854                         time.sleep(5)
 855                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 856
 857                 # Reboot Outlet  N        (Y/N)?
 858                 if dryrun:
 859                         self.ifThenSend("(Y/N)?", "N")
 860                 else:
 861                         self.ifThenSend("(Y/N)?", "Y")
 862                 time.sleep(5)
 863                 self.ifThenSend("DS-RPC>", "")
 864
 865                 self.close()
 866                 return 0
 867
 868 class WTIIPS4(PCUControl):
 869         def run(self, node_port, dryrun):
 870                 self.open(self.host)
 871                 self.sendPassword(self.password, "Enter Password:")
 872
 873                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 874                 if not dryrun:
 875                         self.ifThenSend("Sure? (Y/N): ", "N")
 876                 else:
 877                         self.ifThenSend("Sure? (Y/N): ", "Y")
 878
 879                 self.ifThenSend("IPS> ", "")
 880
 881                 self.close()
 882                 return 0
 883
 884 class ePowerSwitchGood(PCUControl):
 885         # NOTE:
 886         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
 887         #               For some reason this both doesn't work and in some cases, actually
 888         #               hangs the PCU.  Definitely not what we want.
 889         #
 890         #               The code below is much simpler.  Just letting things fail first,
 891         #               and then, trying again with authentication string in the header.
 892         #
 893         def run(self, node_port, dryrun):
 894                 self.transport = None
 895                 self.url = "http://%s:%d/" % (self.host,80)
 896                 uri = "%s:%d" % (self.host,80)
 897
 898                 req = urllib2.Request(self.url)
 899                 try:
 900                         handle = urllib2.urlopen(req)
 901                 except IOError, e:
 902                         # NOTE: this is expected to fail initially
 903                         pass
 904                 else:
 905                         print self.url
 906                         print "-----------"
 907                         print handle.read()
 908                         print "-----------"
 909                         return "ERROR: not protected by HTTP authentication"
 910
 911                 if not hasattr(e, 'code') or e.code != 401:
 912                         return "ERROR: failed for: %s" % str(e)
 913
 914                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
 915                 # NOTE: assuming basic realm authentication.
 916                 authheader = "Basic %s" % base64data
 917                 req.add_header("Authorization", authheader)
 918
 919                 try:
 920                         f = urllib2.urlopen(req)
 921                 except IOError, e:
 922                         # failing here means the User/passwd is wrong (hopefully)
 923                         raise ExceptionPassword("Incorrect username/password")
 924
 925                 # NOTE: after verifying that the user/password is correct,
 926                 #               actually reboot the given node.
 927                 if not dryrun:
 928                         try:
 929                                 data = urllib.urlencode({'P%d' % node_port : "r"})
 930                                 req = urllib2.Request(self.url + "cmd.html")
 931                                 req.add_header("Authorization", authheader)
 932                                 # add data to handler,
 933                                 f = urllib2.urlopen(req, data)
 934                                 if self.verbose: print f.read()
 935                         except:
 936                                 import traceback; traceback.print_exc()
 937                                 from nodecommon import email_exception
 938                                 email_exception()
 939
 940                                 # fetch url one more time on cmd.html, econtrol.html or whatever.
 941                                 # pass
 942                 else:
 943                         if self.verbose: print f.read()
 944
 945                 self.close()
 946                 return 0
 947
 948 class CustomPCU(PCUControl):
 949         def run(self, node_port, dryrun):
 950                 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
 951
 952                 if not dryrun:
 953                         # Turn host off, then on
 954                         formstr = "plab%s=off" % node_port
 955                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 956                         time.sleep(5)
 957                         formstr = "plab%s=on" % node_port
 958                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 959                 else:
 960                         os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
 961
 962
 963 class ePowerSwitchOld(PCUControl):
 964         def run(self, node_port, dryrun):
 965                 self.url = "http://%s:%d/" % (self.host,80)
 966                 uri = "%s:%d" % (self.host,80)
 967
 968                 # create authinfo
 969                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 970                 authinfo.add_password (None, uri, self.username, self.password)
 971                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 972
 973                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 974                 transport = urllib2.build_opener(authinfo)
 975                 f = transport.open(self.url)
 976                 if self.verbose: print f.read()
 977
 978                 if not dryrun:
 979                         transport = urllib2.build_opener(authhandler)
 980                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
 981                         if self.verbose: print f.read()
 982
 983                 self.close()
 984                 return 0
 985
 986 class ePowerSwitch(PCUControl):
 987         def run(self, node_port, dryrun):
 988                 self.url = "http://%s:%d/" % (self.host,80)
 989                 uri = "%s:%d" % (self.host,80)
 990
 991                 # TODO: I'm still not sure what the deal is here.
 992                 #               two independent calls appear to need to be made before the
 993                 #               reboot will succeed.  It doesn't seem to be possible to do
 994                 #               this with a single call.  I have no idea why.
 995
 996                 # create authinfo
 997                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 998                 authinfo.add_password (None, uri, self.username, self.password)
 999                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1000
1001                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
1002                 transport = urllib2.build_opener()
1003                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
1004                 if self.verbose: print f.read()
1005
1006                 if not dryrun:
1007                         transport = urllib2.build_opener(authhandler)
1008                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
1009                         if self.verbose: print f.read()
1010
1011                 #       data= "P%d=r" % node_port
1012                 #self.open(self.host, self.username, self.password)
1013                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
1014                 #self.sendHTTP("econtrol.html", data)
1015                 #self.sendHTTP("cmd.html", data)
1016
1017                 self.close()
1018                 return 0
1019
1020
1021 ### rebooting european BlackBox PSE boxes
1022 # Thierry Parmentelat - May 11 2005
1023 # tested on 4-ports models known as PSE505-FR
1024 # uses http to POST a data 'P<port>=r'
1025 # relies on basic authentication within http1.0
1026 # first curl-based script was
1027 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
1028 #       http://<hostname>:<http_port>/cmd.html && echo OK
1029
1030 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
1031
1032         global verbose
1033
1034         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
1035         data= "P%d=r" % port_in_pcu
1036         if verbose:
1037                 logger.debug("POSTing '%s' on %s" % (data,url))
1038
1039         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1040         uri = "%s:%d" % (pcu_ip,http_port)
1041         authinfo.add_password (None, uri, username, password)
1042         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1043
1044         opener = urllib2.build_opener(authhandler)
1045         urllib2.install_opener(opener)
1046
1047         if (dryrun):
1048                 return 0
1049
1050         try:
1051                 f = urllib2.urlopen(url,data)
1052
1053                 r= f.read()
1054                 if verbose:
1055                         logger.debug(r)
1056                 return 0
1057
1058         except urllib2.URLError,err:
1059                 logger.info('Could not open http connection', err)
1060                 return "bbpse error"
1061
1062 ### rebooting x10toggle based systems addressed by port
1063 # Marc E. Fiuczynski - May 31 2005
1064 # tested on 4-ports models known as PSE505-FR
1065 # uses ssh and password to login to an account
1066 # that will cause the system to be powercycled.
1067
1068 def x10toggle_reboot(ip, username, password, port, dryrun):
1069         global verbose
1070
1071         ssh = None
1072         try:
1073                 ssh = pyssh.Ssh(username, ip)
1074                 ssh.open()
1075
1076                 # Login
1077                 telnet_answer(ssh, "password:", password)
1078
1079                 if not dryrun:
1080                         # Reboot
1081                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
1082
1083                 # Close
1084                 output = ssh.close()
1085                 if verbose:
1086                         logger.debug(output)
1087                 return 0
1088
1089         except Exception, err:
1090                 if verbose:
1091                         logger.debug(err)
1092                 if ssh:
1093                         output = ssh.close()
1094                         if verbose:
1095                                 logger.debug(output)
1096                 return errno.ETIMEDOUT
1097
1098 ### rebooting Dell systems via RAC card
1099 # Marc E. Fiuczynski - June 01 2005
1100 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1101 #
1102
1103 def runcmd(command, args, username, password, timeout = None):
1104
1105         result = [None]
1106         result_ready = threading.Condition()
1107
1108         def set_result(x):
1109
1110                 result_ready.acquire()
1111                 try:
1112                         result[0] = x
1113                 finally:
1114                         result_ready.notify()
1115                         result_ready.release()
1116
1117         def do_command(command, username, password):
1118
1119                 try:
1120                         # Popen4 is a popen-type class that combines stdout and stderr
1121                         p = popen2.Popen4(command)
1122
1123                         # read all output data
1124                         p.tochild.write("%s\n" % username)
1125                         p.tochild.write("%s\n" % password)
1126                         p.tochild.close()
1127                         data = p.fromchild.read()
1128
1129                         while True:
1130                                 # might get interrupted by a signal in poll() or waitpid()
1131                                 try:
1132                                         retval = p.wait()
1133                                         set_result((retval, data))
1134                                         break
1135                                 except OSError, ex:
1136                                         if ex.errno == errno.EINTR:
1137                                                 continue
1138                                         raise ex
1139                 except Exception, ex:
1140                         set_result(ex)
1141
1142         if args:
1143                 command = " ".join([command] + args)
1144
1145         worker = threading.Thread(target = do_command, args = (command, username, password, ))
1146         worker.setDaemon(True)
1147         result_ready.acquire()
1148         worker.start()
1149         result_ready.wait(timeout)
1150         try:
1151                 if result == [None]:
1152                         raise Exception, "command timed-out: '%s'" % command
1153         finally:
1154                 result_ready.release()
1155         result = result[0]
1156
1157         if isinstance(result, Exception):
1158                 raise result
1159         else:
1160                 (retval, data) = result
1161                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1162                         return data
1163                 else:
1164                         out = "system command ('%s') " % command
1165                         if os.WIFEXITED(retval):
1166                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1167                         else:
1168                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1169                         if data:
1170                                 out += "; output follows:\n" + data
1171                         raise Exception, out
1172
1173 def racadm_reboot(host, username, password, port, dryrun):
1174         global verbose
1175
1176         ip = socket.gethostbyname(host)
1177         try:
1178                 cmd = "/usr/sbin/racadm"
1179                 os.stat(cmd)
1180                 if not dryrun:
1181                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1182                                 username, password)
1183                 else:
1184                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1185                                 username, password)
1186
1187                 print "RUNCMD: %s" % output
1188                 if verbose:
1189                         logger.debug(output)
1190                 return 0
1191
1192         except Exception, err:
1193                 logger.debug("runcmd raised exception %s" % err)
1194                 if verbose:
1195                         logger.debug(err)
1196                 return -1
1197
1198 def pcu_name(pcu):
1199         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1200                 return pcu['hostname']
1201         elif pcu['ip'] is not None and pcu['ip'] is not "":
1202                 return pcu['ip']
1203         else:
1204                 return None
1205
1206 #import database
1207 from monitor import database
1208 fb = None
1209
1210 def get_pcu_values(pcu_id):
1211         global fb
1212         if fb == None:
1213                 # this shouldn't be loaded each time...
1214                 fb = database.dbLoad("findbadpcus")
1215
1216         try:
1217                 values = fb['nodes']["id_%s" % pcu_id]['values']
1218         except:
1219                 values = None
1220
1221         return values
1222
1223 def reboot(nodename):
1224         return reboot_policy(nodename, True, False)
1225
1226 def reboot_policy(nodename, continue_probe, dryrun):
1227         global verbose
1228
1229         pcu = plc.getpcu(nodename)
1230         if not pcu:
1231                 logger.debug("no pcu for %s" % nodename)
1232                 print "no pcu for %s" % nodename
1233                 return False # "%s has no pcu" % nodename
1234
1235         values = get_pcu_values(pcu['pcu_id'])
1236         if values == None:
1237                 logger.debug("No values for pcu probe %s" % nodename)
1238                 print "No values for pcu probe %s" % nodename
1239                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1240
1241         # Try the PCU first
1242         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1243
1244         ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1245
1246         if ret != 0:
1247                 print ret
1248                 return False
1249         else:
1250                 print "return true"
1251                 return True
1252
1253 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1254         rb_ret = ""
1255
1256         try:
1257                 # DataProbe iPal (many sites)
1258                 if  continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1259                         ipal = IPAL(values, verbose, ['23', '80', '9100'])
1260                         rb_ret = ipal.reboot(values[nodename], dryrun)
1261
1262                 # APC Masterswitch (Berkeley)
1263                 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1264                                                                   values['model'].find("Masterswitch") >= 0 ):
1265                         print values
1266
1267                         # TODO: make a more robust version of APC
1268                         if values['pcu_id'] in [1102,1163,1055,1111,1231,1113,1127,1128,1148]:
1269                                 apc = APCEurope(values, verbose, ['22', '23'])
1270                                 rb_ret = apc.reboot(values[nodename], dryrun)
1271
1272                         elif values['pcu_id'] in [1110,86]:
1273                                 apc = APCBrazil(values, verbose, ['22', '23'])
1274                                 rb_ret = apc.reboot(values[nodename], dryrun)
1275
1276                         elif values['pcu_id'] in [1221,1225,1220]:
1277                                 apc = APCBerlin(values, verbose, ['22', '23'])
1278                                 rb_ret = apc.reboot(values[nodename], dryrun)
1279
1280                         elif values['pcu_id'] in [1173,1240,47]:
1281                                 apc = APCFolsom(values, verbose, ['22', '23'])
1282                                 rb_ret = apc.reboot(values[nodename], dryrun)
1283
1284                         else:
1285                                 apc = APCMaster(values, verbose, ['22', '23'])
1286                                 rb_ret = apc.reboot(values[nodename], dryrun)
1287
1288                 # BayTech DS4-RPC
1289                 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1290                         if values['pcu_id'] in [1056,1237,1052,1209,1002,1008,1041,1013,1022]:
1291                                 # These  require a 'ctrl-c' to be sent...
1292                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1293                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1294
1295                         elif values['pcu_id'] in [93]:
1296                                 baytech = BayTechAU(values, verbose, ['22', '23'])
1297                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1298
1299                         elif values['pcu_id'] in [1057]:
1300                                 # These  require a 'ctrl-c' to be sent...
1301                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1302                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1303
1304                         elif values['pcu_id'] in [1012]:
1305                                 # This pcu sometimes doesn't present the 'Username' prompt,
1306                                 # unless you immediately try again...
1307                                 try:
1308                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1309                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1310                                 except:
1311                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1312                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1313                         else:
1314                                 baytech = BayTech(values, verbose, ['22', '23'])
1315                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1316
1317                 # iLO
1318                 elif continue_probe and values['model'].find("ilo") >= 0:
1319                         try:
1320                                 hpilo = HPiLO(values, verbose, ['22'])
1321                                 rb_ret = hpilo.reboot(0, dryrun)
1322                                 if rb_ret != 0:
1323                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1324                                         rb_ret = hpilo.reboot(0, dryrun)
1325                         except:
1326                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1327                                 rb_ret = hpilo.reboot(0, dryrun)
1328
1329                 # DRAC ssh
1330                 elif continue_probe and values['model'].find("DRAC") >= 0:
1331                         # TODO: I don't think DRACRacAdm will throw an exception for the
1332                         # default method to catch...
1333                         try:
1334                                 if values['pcu_id'] in [1402]:
1335                                         drac = DRAC(values, verbose, ['22'])
1336                                         rb_ret = drac.reboot(0, dryrun)
1337                                 else:
1338                                         drac = DRACRacAdm(values, verbose, ['443', '5869'])
1339                                         rb_ret = drac.reboot(0, dryrun)
1340                         except:
1341                                 drac = DRAC(values, verbose, ['22'])
1342                                 rb_ret = drac.reboot(0, dryrun)
1343
1344                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1345                                 wti = WTIIPS4(values, verbose, ['23'])
1346                                 rb_ret = wti.reboot(values[nodename], dryrun)
1347
1348                 elif continue_probe and values['model'].find("AMT") >= 0:
1349                                 amt = IntelAMT(values, verbose, ['16992'])
1350                                 rb_ret = amt.reboot(values[nodename], dryrun)
1351
1352                 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1353                 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1354                         # TODO: allow a different port than http 80.
1355                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1356                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1357                         elif values['pcu_id'] in [1003]:
1358                                 # OLD EPOWER
1359                                 print "OLD EPOWER"
1360                                 eps = ePowerSwitch(values, verbose, ['80'])
1361                         else:
1362                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1363
1364                         rb_ret = eps.reboot(values[nodename], dryrun)
1365                 elif continue_probe and values['pcu_id'] in [1122]:
1366                         custom = CustomPCU(values, verbose, ['80', '443'])
1367                         custom.reboot(values[nodename], dryrun)
1368
1369                 elif continue_probe:
1370                         rb_ret = "Unsupported_PCU"
1371
1372                 elif continue_probe == False:
1373                         if 'portstatus' in values:
1374                                 rb_ret = "NetDown"
1375                         else:
1376                                 rb_ret = "Not_Run"
1377                 else:
1378                         rb_ret = -1
1379
1380         except ExceptionPort, err:
1381                 rb_ret = str(err)
1382
1383         return rb_ret
1384         # ????
1385         #elif continue_probe and values['protocol'] == "racadm" and \
1386         #               values['model'] == "RAC":
1387         #       rb_ret = racadm_reboot(pcu_name(values),
1388         #                                                                 values['username'],
1389         #                                                                 values['password'],
1390         #                                                                 pcu[nodename],
1391         #                                                                 dryrun)
1392
1393 def main():
1394         logger.setLevel(logging.DEBUG)
1395         ch = logging.StreamHandler()
1396         ch.setLevel(logging.DEBUG)
1397         formatter = logging.Formatter('LOGGER - %(message)s')
1398         ch.setFormatter(formatter)
1399         logger.addHandler(ch)
1400
1401         try:
1402                 if "test" in sys.argv:
1403                         dryrun = True
1404                 else:
1405                         dryrun = False
1406
1407                 for node in sys.argv[1:]:
1408                         if node == "test": continue
1409
1410                         print "Rebooting %s" % node
1411                         if reboot_policy(node, True, dryrun):
1412                                 print "success"
1413                         else:
1414                                 print "failed"
1415         except Exception, err:
1416                 import traceback; traceback.print_exc()
1417                 from nodecommon import email_exception
1418                 email_exception()
1419                 print err
1420
1421 if __name__ == '__main__':
1422         import plc
1423         logger = logging.getLogger("monitor")
1424         main()