reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import urllib
  12 import threading, popen2
  13 import array, struct
  14 import plc
  15 import base64
  16 from subprocess import PIPE, Popen
  17 import ssh.pxssh as pxssh
  18 import ssh.pexpect as pexpect
  19 import socket
  20 import moncommands
  21
  22 # Use our versions of telnetlib and pyssh
  23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  24 import telnetlib
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  26 import pyssh
  27
  28 # Timeouts in seconds
  29 TELNET_TIMEOUT = 45
  30
  31 # Event class ID from pcu events
  32 #NODE_POWER_CONTROL = 3
  33
  34 # Monitor user ID
  35 #MONITOR_USER_ID = 11142
  36
  37 import logging
  38 logger = logging.getLogger("monitor")
  39 verbose = 1
  40 #dryrun = 0;
  41
  42 class ExceptionNoTransport(Exception): pass
  43 class ExceptionNotFound(Exception): pass
  44 class ExceptionPassword(Exception): pass
  45 class ExceptionTimeout(Exception): pass
  46 class ExceptionPrompt(Exception): pass
  47 class ExceptionSequence(Exception): pass
  48 class ExceptionReset(Exception): pass
  49 class ExceptionPort(Exception): pass
  50 class ExceptionUsername(Exception): pass
  51
  52 def telnet_answer(telnet, expected, buffer):
  53         global verbose
  54
  55         output = telnet.read_until(expected, TELNET_TIMEOUT)
  56         #if verbose:
  57         #       logger.debug(output)
  58         if output.find(expected) == -1:
  59                 raise ExceptionNotFound, "'%s' not found" % expected
  60         else:
  61                 telnet.write(buffer + "\r\n")
  62
  63
  64 # PCU has model, host, preferred-port, user, passwd,
  65
  66 # This is an object derived directly form the PLCAPI DB fields
  67 class PCU(object):
  68         def __init__(self, plc_pcu_dict):
  69                 for field in ['username', 'password', 'site_id',
  70                                                 'hostname', 'ip',
  71                                                 'pcu_id', 'model',
  72                                                 'node_ids', 'ports', ]:
  73                         if field in plc_pcu_dict:
  74                                 self.__setattr__(field, plc_pcu_dict[field])
  75                         else:
  76                                 raise Exception("No such field %s in PCU object" % field)
  77
  78 # These are the convenience functions build around the PCU object.
  79 class PCUModel(PCU):
  80         def __init__(self, plc_pcu_dict):
  81                 PCU.__init__(self, plc_pcu_dict)
  82                 self.host = self.pcu_name()
  83
  84         def pcu_name(self):
  85                 if self.hostname is not None and self.hostname is not "":
  86                         return self.hostname
  87                 elif self.ip is not None and self.ip is not "":
  88                         return self.ip
  89                 else:
  90                         return None
  91
  92         def nodeidToPort(self, node_id):
  93                 if node_id in self.node_ids:
  94                         for i in range(0, len(self.node_ids)):
  95                                 if node_id == self.node_ids[i]:
  96                                         return self.ports[i]
  97
  98                 raise Exception("No such Node ID: %d" % node_id)
  99
 100 # This class captures the observed pcu records from FindBadPCUs.py
 101 class PCURecord:
 102         def __init__(self, pcu_record_dict):
 103                 for field in ['nodenames', 'portstatus',
 104                                                 'dnsmatch',
 105                                                 'complete_entry', ]:
 106                         if field in pcu_record_dict:
 107                                 if field == "reboot":
 108                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 109                                 else:
 110                                         self.__setattr__(field, pcu_record_dict[field])
 111                         else:
 112                                 raise Exception("No such field %s in pcu record dict" % field)
 113
 114 class Transport:
 115         TELNET = 1
 116         SSH    = 2
 117         HTTP   = 3
 118         IPAL   = 4
 119
 120         TELNET_TIMEOUT = 60
 121
 122         def __init__(self, type, verbose):
 123                 self.type = type
 124                 self.verbose = verbose
 125                 self.transport = None
 126
 127         def open(self, host, username=None, password=None, prompt="User Name"):
 128                 transport = None
 129
 130                 if self.type == self.TELNET:
 131                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 132                         transport.set_debuglevel(self.verbose)
 133                         if username is not None:
 134                                 self.transport = transport
 135                                 self.ifThenSend(prompt, username, ExceptionUsername)
 136
 137                 elif self.type == self.SSH:
 138                         if username is not None:
 139                                 transport = pyssh.Ssh(username, host)
 140                                 transport.set_debuglevel(self.verbose)
 141                                 transport.open()
 142                                 # TODO: have an ssh set_debuglevel() also...
 143                         else:
 144                                 raise Exception("Username cannot be None for ssh transport.")
 145                 elif self.type == self.HTTP:
 146                         self.url = "http://%s:%d/" % (host,80)
 147                         uri = "%s:%d" % (host,80)
 148
 149                         # create authinfo
 150                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 151                         authinfo.add_password (None, uri, username, password)
 152                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 153
 154                         transport = urllib2.build_opener(authhandler)
 155
 156                 else:
 157                         raise Exception("Unknown transport type: %s" % self.type)
 158
 159                 self.transport = transport
 160                 return True
 161
 162         def close(self):
 163                 if self.type == self.TELNET:
 164                         self.transport.close()
 165                 elif self.type == self.SSH:
 166                         self.transport.close()
 167                 elif self.type == self.HTTP:
 168                         pass
 169                 else:
 170                         raise Exception("Unknown transport type %s" % self.type)
 171                 self.transport = None
 172
 173         def sendHTTP(self, resource, data):
 174                 if self.verbose:
 175                         print "POSTing '%s' to %s" % (data,self.url + resource)
 176
 177                 try:
 178                         f = self.transport.open(self.url + resource ,data)
 179                         r = f.read()
 180                         if self.verbose:
 181                                 print r
 182
 183                 except urllib2.URLError,err:
 184                         logger.info('Could not open http connection', err)
 185                         return "http transport error"
 186
 187                 return 0
 188
 189         def sendPassword(self, password, prompt=None):
 190                 if self.type == self.TELNET:
 191                         if prompt == None:
 192                                 self.ifThenSend("Password", password, ExceptionPassword)
 193                         else:
 194                                 self.ifThenSend(prompt, password, ExceptionPassword)
 195                 elif self.type == self.SSH:
 196                         self.ifThenSend("password:", password, ExceptionPassword)
 197                 elif self.type == self.HTTP:
 198                         pass
 199                 else:
 200                         raise Exception("Unknown transport type: %s" % self.type)
 201
 202         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 203
 204                 if self.transport != None:
 205                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 206                         if output.find(expected) == -1:
 207                                 raise ErrorClass, "'%s' not found" % expected
 208                         else:
 209                                 self.transport.write(buffer + "\r\n")
 210                 else:
 211                         raise ExceptionNoTransport("transport object is type None")
 212
 213         def ifElse(self, expected, ErrorClass):
 214                 try:
 215                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 216                 except:
 217                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 218
 219
 220 class PCUControl(Transport,PCUModel,PCURecord):
 221         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 222                 PCUModel.__init__(self, plc_pcu_record)
 223                 PCURecord.__init__(self, plc_pcu_record)
 224                 type = None
 225                 if self.portstatus:
 226                         if '22' in supported_ports and self.portstatus['22'] == "open":
 227                                 type = Transport.SSH
 228                         elif '23' in supported_ports and self.portstatus['23'] == "open":
 229                                 type = Transport.TELNET
 230                         elif '80' in supported_ports and self.portstatus['80'] == "open":
 231                                 type = Transport.HTTP
 232                         elif '443' in supported_ports and self.portstatus['443'] == "open":
 233                                 type = Transport.HTTP
 234                         elif '5869' in supported_ports and self.portstatus['5869'] == "open":
 235                                 # For DRAC cards. Racadm opens this port.
 236                                 type = Transport.HTTP
 237                         elif '9100' in supported_ports and self.portstatus['9100'] == "open":
 238                                 type = Transport.IPAL
 239                         elif '16992' in supported_ports and self.portstatus['16992'] == "open":
 240                                 type = Transport.HTTP
 241                         else:
 242                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 243                 else:
 244                         raise Exception("No Portstatus: No transport because no open ports")
 245                 Transport.__init__(self, type, verbose)
 246
 247         def run(self, node_port, dryrun):
 248                 """ This function is to be defined by the specific PCU instance.  """
 249                 pass
 250
 251         def reboot(self, node_port, dryrun):
 252                 try:
 253                         return self.run(node_port, dryrun)
 254                 except ExceptionNotFound, err:
 255                         return "error: " + str(err)
 256                 except ExceptionPassword, err:
 257                         return "password exception: " + str(err)
 258                 except ExceptionTimeout, err:
 259                         return "timeout exception: " + str(err)
 260                 except ExceptionUsername, err:
 261                         return "exception: no username prompt: " + str(err)
 262                 except ExceptionSequence, err:
 263                         return "sequence error: " + str(err)
 264                 except ExceptionPrompt, err:
 265                         return "prompt exception: " + str(err)
 266                 except ExceptionPort, err:
 267                         return "no ports exception: " + str(err)
 268                 except socket.error, err:
 269                         return "socket error: timeout: " + str(err)
 270                 except EOFError, err:
 271                         if self.verbose:
 272                                 logger.debug("reboot: EOF")
 273                                 logger.debug(err)
 274                         self.transport.close()
 275                         import traceback
 276                         traceback.print_exc()
 277                         return "EOF connection reset" + str(err)
 278                 except:
 279                         from nodecommon import email_exception
 280                         email_exception()
 281                         raise Exception('unknown')
 282
 283 class IPAL(PCUControl):
 284         """
 285                 This now uses a proprietary format for communicating with the PCU.  I
 286                 prefer it to Telnet, and Web access, since it's much lighter weight
 287                 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
 288         """
 289
 290         def format_msg(self, data, cmd):
 291                 esc = chr(int('1b',16))
 292                 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
 293
 294         def recv_noblock(self, s, count):
 295                 import errno
 296
 297                 try:
 298                         # TODO: make sleep backoff, before stopping.
 299                         time.sleep(4)
 300                         ret = s.recv(count, socket.MSG_DONTWAIT)
 301                 except socket.error, e:
 302                         if e[0] == errno.EAGAIN:
 303                                 raise Exception(e[1])
 304                         else:
 305                                 # TODO: not other exceptions.
 306                                 raise Exception(e)
 307                 return ret
 308
 309         def run(self, node_port, dryrun):
 310                 import errno
 311
 312                 power_on = False
 313
 314                 print "open socket"
 315                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 316                 try:
 317                         print "connect"
 318                         s.connect((self.host, 9100))
 319                 except socket.error, e:
 320                         s.close()
 321                         if e[0] == errno.ECONNREFUSED:
 322                                 # cannot connect to remote host
 323                                 raise Exception(e[1])
 324                         else:
 325                                 # TODO: what other conditions are there?
 326                                 raise Exception(e)
 327
 328                 # get current status
 329                 print "Checking status"
 330                 s.send(self.format_msg("", 'O'))
 331                 ret = self.recv_noblock(s, 8)
 332                 print "Current status is '%s'" % ret
 333
 334                 if ret == '':
 335                         raise Exception("Status returned 'another session already open' %s : %s" % (node_port, ret))
 336
 337
 338                 if node_port < len(ret):
 339                         status = ret[node_port]
 340                         if status == '1':
 341                                 # up
 342                                 power_on = True
 343                         elif status == '0':
 344                                 # down
 345                                 power_on = False
 346                         else:
 347                                 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 348                 else:
 349                         raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 350
 351
 352                 if not dryrun:
 353                         if power_on:
 354                                 print "Pulsing %s" % node_port
 355                                 s.send(self.format_msg("%s" % node_port, 'P'))
 356                         else:
 357                                 # NOTE: turn power on ; do not pulse the port.
 358                                 print "Power was off, so turning on ..."
 359                                 s.send(self.format_msg("%s" % node_port, 'E'))
 360                                 #s.send(self.format_msg("%s" % node_port, 'P'))
 361
 362                         print "Receiving response."
 363                         ret = self.recv_noblock(s, 8)
 364                         print "Current status is '%s'" % ret
 365
 366                         if node_port < len(ret):
 367                                 status = ret[node_port]
 368                                 if status == '1':
 369                                         # up
 370                                         power_on = True
 371                                 elif status == '0':
 372                                         # down
 373                                         power_on = False
 374                                 else:
 375                                         raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 376                         else:
 377                                 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 378
 379                         if power_on:
 380                                 return 0
 381                         else:
 382                                 return "Failed Power On"
 383
 384                 s.close()
 385                 return 0
 386
 387 # TELNET version of protocol...
 388 #               #self.open(self.host)
 389 #               ## XXX Some iPals require you to hit Enter a few times first
 390 #               #self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 391 #               # Login
 392 #               self.ifThenSend("Password >", self.password, ExceptionPassword)
 393 #               self.transport.write("\r\n\r\n")
 394 #               if not dryrun: # P# - Pulse relay
 395 #                       print "node_port %s" % node_port
 396 #                       self.ifThenSend("Enter >",
 397 #                                                       "P7", # % node_port,
 398 #                                                       ExceptionNotFound)
 399 #                       print "send newlines"
 400 #                       self.transport.write("\r\n\r\n")
 401 #                       print "after new lines"
 402 #               # Get the next prompt
 403 #               print "wait for enter"
 404 #               self.ifElse("Enter >", ExceptionTimeout)
 405 #               print "closing "
 406 #               self.close()
 407 #               return 0
 408
 409 class APCEurope(PCUControl):
 410         def run(self, node_port, dryrun):
 411                 self.open(self.host, self.username)
 412                 self.sendPassword(self.password)
 413
 414                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 415                 self.ifThenSend("\r\n> ", "2")
 416                 self.ifThenSend("\r\n> ", str(node_port))
 417                 # 3- Immediate Reboot
 418                 self.ifThenSend("\r\n> ", "3")
 419
 420                 if not dryrun:
 421                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 422                                                         "YES\r\n",
 423                                                         ExceptionSequence)
 424                 else:
 425                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 426                                                         "", ExceptionSequence)
 427                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 428
 429                 self.close()
 430                 return 0
 431
 432 class APCBrazil(PCUControl):
 433         def run(self, node_port, dryrun):
 434                 self.open(self.host, self.username)
 435                 self.sendPassword(self.password)
 436
 437                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 438                 self.ifThenSend("\r\n> ", str(node_port))
 439                 # 4- Immediate Reboot
 440                 self.ifThenSend("\r\n> ", "4")
 441
 442                 if not dryrun:
 443                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 444                                                         "YES\r\n",
 445                                                         ExceptionSequence)
 446                 else:
 447                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 448                                                         "", ExceptionSequence)
 449                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 450
 451                 self.close()
 452                 return 0
 453
 454 class APCBerlin(PCUControl):
 455         def run(self, node_port, dryrun):
 456                 self.open(self.host, self.username)
 457                 self.sendPassword(self.password)
 458
 459                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 460                 self.ifThenSend("\r\n> ", "2")
 461                 self.ifThenSend("\r\n> ", "1")
 462                 self.ifThenSend("\r\n> ", str(node_port))
 463                 # 3- Immediate Reboot
 464                 self.ifThenSend("\r\n> ", "3")
 465
 466                 if not dryrun:
 467                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 468                                                         "YES\r\n",
 469                                                         ExceptionSequence)
 470                 else:
 471                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 472                                                         "", ExceptionSequence)
 473                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 474
 475                 self.close()
 476                 return 0
 477
 478 class APCFolsom(PCUControl):
 479         def run(self, node_port, dryrun):
 480                 self.open(self.host, self.username)
 481                 self.sendPassword(self.password)
 482
 483                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 484                 self.ifThenSend("\r\n> ", "2")
 485                 self.ifThenSend("\r\n> ", "1")
 486                 self.ifThenSend("\r\n> ", str(node_port))
 487                 self.ifThenSend("\r\n> ", "1")
 488
 489                 # 3- Immediate Reboot
 490                 self.ifThenSend("\r\n> ", "3")
 491
 492                 if not dryrun:
 493                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 494                                                         "YES\r\n",
 495                                                         ExceptionSequence)
 496                 else:
 497                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 498                                                         "", ExceptionSequence)
 499                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 500
 501                 self.close()
 502                 return 0
 503
 504 class APCMaster(PCUControl):
 505         def run(self, node_port, dryrun):
 506                 print "Rebooting %s" % self.host
 507                 self.open(self.host, self.username)
 508                 self.sendPassword(self.password)
 509
 510                 # 1- Device Manager
 511                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 512                 # 3- Outlet Control/Config
 513                 self.ifThenSend("\r\n> ", "3")
 514                 # n- Outlet n
 515                 self.ifThenSend("\r\n> ", str(node_port))
 516                 # 1- Control Outlet
 517                 self.ifThenSend("\r\n> ", "1")
 518                 # 3- Immediate Reboot
 519                 self.ifThenSend("\r\n> ", "3")
 520
 521                 if not dryrun:
 522                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 523                                                         "YES\r\n",
 524                                                         ExceptionSequence)
 525                 else:
 526                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 527                                                         "", ExceptionSequence)
 528                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 529
 530                 self.close()
 531                 return 0
 532
 533 class APC(PCUControl):
 534         def __init__(self, plc_pcu_record, verbose):
 535                 PCUControl.__init__(self, plc_pcu_record, verbose)
 536
 537                 self.master = APCMaster(plc_pcu_record, verbose)
 538                 self.folsom = APCFolsom(plc_pcu_record, verbose)
 539                 self.europe = APCEurope(plc_pcu_record, verbose)
 540
 541         def run(self, node_port, dryrun):
 542                 try_again = True
 543                 sleep_time = 1
 544
 545                 for pcu in [self.master, self.europe, self.folsom]:
 546                         if try_again:
 547                                 try:
 548                                         print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
 549                                         try_again = False
 550                                         print "sleeping 5"
 551                                         time.sleep(sleep_time)
 552                                         ret = pcu.reboot(node_port, dryrun)
 553                                 except ExceptionSequence, err:
 554                                         del pcu
 555                                         sleep_time = 130
 556                                         try_again = True
 557
 558                 if try_again:
 559                         return "Unknown reboot sequence for APC PCU"
 560                 else:
 561                         return ret
 562
 563 class IntelAMT(PCUControl):
 564         def run(self, node_port, dryrun):
 565
 566                 cmd = moncommands.CMD()
 567                 #[cmd_str = "IntelAMTSDK/Samples/RemoteControl/remoteControl"
 568                 cmd_str = "cmdamt/remoteControl"
 569
 570                 if dryrun:
 571                         # NOTE: -p checks the power state of the host.
 572                         # TODO: parse the output to find out if it's ok or not.
 573                         cmd_str += " -p http://%s:16992/RemoteControlService  -user admin -pass '%s' " % (self.host, self.password )
 574                 else:
 575                         cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
 576
 577                 print cmd_str
 578                 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
 579
 580 class DRACRacAdm(PCUControl):
 581         def run(self, node_port, dryrun):
 582
 583                 print "trying racadm_reboot..."
 584                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 585
 586                 return 0
 587
 588 class DRAC(PCUControl):
 589         def run(self, node_port, dryrun):
 590                 self.open(self.host, self.username)
 591                 self.sendPassword(self.password)
 592
 593                 print "logging in..."
 594                 self.transport.write("\r\n")
 595                 # Testing Reboot ?
 596                 if dryrun:
 597                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 598                 else:
 599                         # Reset this machine
 600                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 601
 602                 self.ifThenSend("[%s]#" % self.username, "exit")
 603
 604                 self.close()
 605                 return 0
 606
 607 class HPiLO(PCUControl):
 608         def run(self, node_port, dryrun):
 609                 self.open(self.host, self.username)
 610                 self.sendPassword(self.password)
 611
 612                 # </>hpiLO->
 613                 self.ifThenSend("</>hpiLO->", "cd system1")
 614
 615                 # Reboot Outlet  N        (Y/N)?
 616                 if dryrun:
 617                         self.ifThenSend("</system1>hpiLO->", "POWER")
 618                 else:
 619                         # Reset this machine
 620                         self.ifThenSend("</system1>hpiLO->", "reset")
 621
 622                 self.ifThenSend("</system1>hpiLO->", "exit")
 623
 624                 self.close()
 625                 return 0
 626
 627
 628 class HPiLOHttps(PCUControl):
 629         def run(self, node_port, dryrun):
 630
 631                 locfg = moncommands.CMD()
 632                 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 633                                         self.host, "iloxml/Get_Network.xml",
 634                                         self.username, self.password)
 635                 sout, serr = locfg.run_noexcept(cmd)
 636
 637                 if sout.strip() != "":
 638                         print "sout: %s" % sout.strip()
 639                         return sout.strip()
 640
 641                 if not dryrun:
 642                         locfg = moncommands.CMD()
 643                         cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 644                                                 self.host, "iloxml/Reset_Server.xml",
 645                                                 self.username, self.password)
 646                         sout, serr = locfg.run_noexcept(cmd)
 647
 648                         if sout.strip() != "":
 649                                 print "sout: %s" % sout.strip()
 650                                 #return sout.strip()
 651                 return 0
 652
 653 class BayTechAU(PCUControl):
 654         def run(self, node_port, dryrun):
 655                 self.open(self.host, self.username, None, "Enter user name:")
 656                 self.sendPassword(self.password, "Enter Password:")
 657
 658                 #self.ifThenSend("RPC-16>", "Status")
 659                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 660
 661                 # Reboot Outlet  N        (Y/N)?
 662                 if dryrun:
 663                         self.ifThenSend("(Y/N)?", "N")
 664                 else:
 665                         self.ifThenSend("(Y/N)?", "Y")
 666                 self.ifThenSend("RPC3-NC>", "")
 667
 668                 self.close()
 669                 return 0
 670
 671 class BayTechGeorgeTown(PCUControl):
 672         def run(self, node_port, dryrun):
 673                 # this initial open/close is to prevent things from raising an
 674                 # exception.  the pcu always is weird during the first connection, and
 675                 # even if it's not, what does it matter to open a second connection
 676                 # right away?
 677                 self.open(self.host, self.username, None, "Enter user name:")
 678                 self.close()
 679                 time.sleep(1)
 680                 self.open(self.host, self.username, None, "Enter user name:")
 681                 self.sendPassword(self.password, "Enter Password:")
 682
 683                 #self.ifThenSend("RPC-16>", "Status")
 684
 685                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 686
 687                 # Reboot Outlet  N        (Y/N)?
 688                 if dryrun:
 689                         self.ifThenSend("(Y/N)?", "N")
 690                 else:
 691                         self.ifThenSend("(Y/N)?", "Y")
 692                 self.ifThenSend("RPC-16>", "")
 693
 694                 self.close()
 695                 return 0
 696
 697 class BayTechCtrlCUnibe(PCUControl):
 698         """
 699                 For some reason, these units let you log in fine, but they hang
 700                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 701                 why.
 702         """
 703         def run(self, node_port, dryrun):
 704                 print "BayTechCtrlC %s" % self.host
 705
 706                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 707                 s = pxssh.pxssh()
 708                 if not s.login(self.host, self.username, self.password, ssh_options):
 709                         raise ExceptionPassword("Invalid Password")
 710                 # Otherwise, the login succeeded.
 711
 712                 # Send a ctrl-c to the remote process.
 713                 print "sending ctrl-c"
 714                 s.send(chr(3))
 715
 716                 # Control Outlets  (5 ,1).........5
 717                 try:
 718                         #index = s.expect("Enter Request")
 719                         index = s.expect(["Enter Request :"])
 720
 721                         if index == 0:
 722                                 print "3"
 723                                 s.send("3\r\n")
 724                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 725                                 if index == 1:
 726                                         s.send(self.username + "\r\n")
 727                                         index = s.expect(["DS-RPC>"])
 728
 729                                 if index == 0:
 730                                         print "Reboot %d" % node_port
 731                                         s.send("Reboot %d\r\n" % node_port)
 732
 733                                         time.sleep(5)
 734                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 735                                         if index == 0:
 736                                                 if dryrun:
 737                                                         print "sending N"
 738                                                         s.send("N\r\n")
 739                                                 else:
 740                                                         print "sending Y"
 741                                                         s.send("Y\r\n")
 742                                         elif index == 1:
 743                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 744                                         elif index == 2:
 745                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 746
 747                                 time.sleep(5)
 748                                 index = s.expect(["DS-RPC>"])
 749                                 #print "got prompt back"
 750
 751                         s.close()
 752
 753                 except pexpect.EOF:
 754                         raise ExceptionPrompt("EOF before expected Prompt")
 755                 except pexpect.TIMEOUT:
 756                         raise ExceptionPrompt("Timeout before expected Prompt")
 757
 758                 return 0
 759
 760 class BayTechCtrlC(PCUControl):
 761         """
 762                 For some reason, these units let you log in fine, but they hang
 763                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 764                 why.
 765         """
 766         def run(self, node_port, dryrun):
 767                 print "BayTechCtrlC %s" % self.host
 768
 769                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 770                 s = pxssh.pxssh()
 771                 if not s.login(self.host, self.username, self.password, ssh_options):
 772                         raise ExceptionPassword("Invalid Password")
 773                 # Otherwise, the login succeeded.
 774
 775                 # Send a ctrl-c to the remote process.
 776                 print "SENDING ctrl-c"
 777                 s.send(chr(3))
 778
 779                 # Control Outlets  (5 ,1).........5
 780                 try:
 781                         print "EXPECTING: ", "Enter Request :"
 782                         index = s.expect(["Enter Request :"])
 783
 784                         if index == 0:
 785                                 print "SENDING: 5"
 786                                 s.send("5\r\n")
 787                                 print "EXPECTING: ", "DS-RPC>"
 788                                 index = s.expect(["DS-RPC>", "Enter user name:", "Port in use."])
 789                                 if index == 1:
 790                                         print "sending username"
 791                                         s.send(self.username + "\r\n")
 792                                         index = s.expect(["DS-RPC>"])
 793                                 elif index == 2:
 794                                         raise ExceptionPrompt("PCU Reported 'Port in use.'")
 795
 796                                 if index == 0:
 797                                         print "SENDING: Reboot %d" % node_port
 798                                         s.send("Reboot %d\r\n" % node_port)
 799
 800                                         print "SLEEPING: 5"
 801                                         time.sleep(5)
 802                                         print "EXPECTING: ", "Y/N?"
 803                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 804                                         if index == 0:
 805                                                 if dryrun:
 806                                                         print "sending N"
 807                                                         s.send("N\r\n")
 808                                                 else:
 809                                                         print "SENDING: Y"
 810                                                         s.send("Y\r\n")
 811                                         elif index == 1:
 812                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 813                                         elif index == 2:
 814                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 815
 816                                 # NOTE: for some reason, the script times out with the
 817                                 # following line.  In manual tests, it works correctly, but
 818                                 # with automated tests, evidently it fails.
 819                                 print "SLEEPING: 5"
 820                                 time.sleep(5)
 821                                 #print "TOTAL--", s.allstr, "--EOT"
 822                                 index = s.expect(["DS-RPC>"])
 823                                 print "got prompt back"
 824
 825                         s.close()
 826
 827                 except pexpect.EOF:
 828                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 829                 except pexpect.TIMEOUT:
 830                         raise ExceptionPrompt("Timeout before Prompt")
 831
 832                 return 0
 833
 834 class BayTech(PCUControl):
 835         def run(self, node_port, dryrun):
 836                 self.open(self.host, self.username)
 837                 self.sendPassword(self.password)
 838
 839                 # Control Outlets  (5 ,1).........5
 840                 self.ifThenSend("Enter Request :", "5")
 841
 842                 # Reboot N
 843                 try:
 844                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 845                 except ExceptionNotFound, msg:
 846                         # one machine is configured to ask for a username,
 847                         # even after login...
 848                         print "msg: %s" % msg
 849                         self.transport.write(self.username + "\r\n")
 850                         time.sleep(5)
 851                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 852
 853                 # Reboot Outlet  N        (Y/N)?
 854                 if dryrun:
 855                         self.ifThenSend("(Y/N)?", "N")
 856                 else:
 857                         self.ifThenSend("(Y/N)?", "Y")
 858                 time.sleep(5)
 859                 self.ifThenSend("DS-RPC>", "")
 860
 861                 self.close()
 862                 return 0
 863
 864 class WTIIPS4(PCUControl):
 865         def run(self, node_port, dryrun):
 866                 self.open(self.host)
 867                 self.sendPassword(self.password, "Enter Password:")
 868
 869                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 870                 if not dryrun:
 871                         self.ifThenSend("Sure? (Y/N): ", "N")
 872                 else:
 873                         self.ifThenSend("Sure? (Y/N): ", "Y")
 874
 875                 self.ifThenSend("IPS> ", "")
 876
 877                 self.close()
 878                 return 0
 879
 880 class ePowerSwitchGood(PCUControl):
 881         # NOTE:
 882         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
 883         #               For some reason this both doesn't work and in some cases, actually
 884         #               hangs the PCU.  Definitely not what we want.
 885         #
 886         #               The code below is much simpler.  Just letting things fail first,
 887         #               and then, trying again with authentication string in the header.
 888         #
 889         def run(self, node_port, dryrun):
 890                 self.transport = None
 891                 self.url = "http://%s:%d/" % (self.host,80)
 892                 uri = "%s:%d" % (self.host,80)
 893
 894                 req = urllib2.Request(self.url)
 895                 try:
 896                         handle = urllib2.urlopen(req)
 897                 except IOError, e:
 898                         # NOTE: this is expected to fail initially
 899                         pass
 900                 else:
 901                         print self.url
 902                         print "-----------"
 903                         print handle.read()
 904                         print "-----------"
 905                         return "ERROR: not protected by HTTP authentication"
 906
 907                 if not hasattr(e, 'code') or e.code != 401:
 908                         return "ERROR: failed for: %s" % str(e)
 909
 910                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
 911                 # NOTE: assuming basic realm authentication.
 912                 authheader = "Basic %s" % base64data
 913                 req.add_header("Authorization", authheader)
 914
 915                 try:
 916                         f = urllib2.urlopen(req)
 917                 except IOError, e:
 918                         # failing here means the User/passwd is wrong (hopefully)
 919                         raise ExceptionPassword("Incorrect username/password")
 920
 921                 # NOTE: after verifying that the user/password is correct,
 922                 #               actually reboot the given node.
 923                 if not dryrun:
 924                         try:
 925                                 data = urllib.urlencode({'P%d' % node_port : "r"})
 926                                 req = urllib2.Request(self.url + "cmd.html")
 927                                 req.add_header("Authorization", authheader)
 928                                 # add data to handler,
 929                                 f = urllib2.urlopen(req, data)
 930                                 if self.verbose: print f.read()
 931                         except:
 932                                 import traceback; traceback.print_exc()
 933                                 from nodecommon import email_exception
 934                                 email_exception()
 935
 936                                 # fetch url one more time on cmd.html, econtrol.html or whatever.
 937                                 # pass
 938                 else:
 939                         if self.verbose: print f.read()
 940
 941                 self.close()
 942                 return 0
 943
 944 class CustomPCU(PCUControl):
 945         def run(self, node_port, dryrun):
 946                 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
 947
 948                 if not dryrun:
 949                         # Turn host off, then on
 950                         formstr = "plab%s=off" % node_port
 951                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 952                         time.sleep(5)
 953                         formstr = "plab%s=on" % node_port
 954                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 955                 else:
 956                         os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
 957
 958
 959 class ePowerSwitchOld(PCUControl):
 960         def run(self, node_port, dryrun):
 961                 self.url = "http://%s:%d/" % (self.host,80)
 962                 uri = "%s:%d" % (self.host,80)
 963
 964                 # create authinfo
 965                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 966                 authinfo.add_password (None, uri, self.username, self.password)
 967                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 968
 969                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 970                 transport = urllib2.build_opener(authinfo)
 971                 f = transport.open(self.url)
 972                 if self.verbose: print f.read()
 973
 974                 if not dryrun:
 975                         transport = urllib2.build_opener(authhandler)
 976                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
 977                         if self.verbose: print f.read()
 978
 979                 self.close()
 980                 return 0
 981
 982 class ePowerSwitch(PCUControl):
 983         def run(self, node_port, dryrun):
 984                 self.url = "http://%s:%d/" % (self.host,80)
 985                 uri = "%s:%d" % (self.host,80)
 986
 987                 # TODO: I'm still not sure what the deal is here.
 988                 #               two independent calls appear to need to be made before the
 989                 #               reboot will succeed.  It doesn't seem to be possible to do
 990                 #               this with a single call.  I have no idea why.
 991
 992                 # create authinfo
 993                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 994                 authinfo.add_password (None, uri, self.username, self.password)
 995                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 996
 997                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 998                 transport = urllib2.build_opener()
 999                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
1000                 if self.verbose: print f.read()
1001
1002                 if not dryrun:
1003                         transport = urllib2.build_opener(authhandler)
1004                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
1005                         if self.verbose: print f.read()
1006
1007                 #       data= "P%d=r" % node_port
1008                 #self.open(self.host, self.username, self.password)
1009                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
1010                 #self.sendHTTP("econtrol.html", data)
1011                 #self.sendHTTP("cmd.html", data)
1012
1013                 self.close()
1014                 return 0
1015
1016
1017 ### rebooting european BlackBox PSE boxes
1018 # Thierry Parmentelat - May 11 2005
1019 # tested on 4-ports models known as PSE505-FR
1020 # uses http to POST a data 'P<port>=r'
1021 # relies on basic authentication within http1.0
1022 # first curl-based script was
1023 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
1024 #       http://<hostname>:<http_port>/cmd.html && echo OK
1025
1026 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
1027
1028         global verbose
1029
1030         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
1031         data= "P%d=r" % port_in_pcu
1032         if verbose:
1033                 logger.debug("POSTing '%s' on %s" % (data,url))
1034
1035         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1036         uri = "%s:%d" % (pcu_ip,http_port)
1037         authinfo.add_password (None, uri, username, password)
1038         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1039
1040         opener = urllib2.build_opener(authhandler)
1041         urllib2.install_opener(opener)
1042
1043         if (dryrun):
1044                 return 0
1045
1046         try:
1047                 f = urllib2.urlopen(url,data)
1048
1049                 r= f.read()
1050                 if verbose:
1051                         logger.debug(r)
1052                 return 0
1053
1054         except urllib2.URLError,err:
1055                 logger.info('Could not open http connection', err)
1056                 return "bbpse error"
1057
1058 ### rebooting x10toggle based systems addressed by port
1059 # Marc E. Fiuczynski - May 31 2005
1060 # tested on 4-ports models known as PSE505-FR
1061 # uses ssh and password to login to an account
1062 # that will cause the system to be powercycled.
1063
1064 def x10toggle_reboot(ip, username, password, port, dryrun):
1065         global verbose
1066
1067         ssh = None
1068         try:
1069                 ssh = pyssh.Ssh(username, ip)
1070                 ssh.open()
1071
1072                 # Login
1073                 telnet_answer(ssh, "password:", password)
1074
1075                 if not dryrun:
1076                         # Reboot
1077                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
1078
1079                 # Close
1080                 output = ssh.close()
1081                 if verbose:
1082                         logger.debug(output)
1083                 return 0
1084
1085         except Exception, err:
1086                 if verbose:
1087                         logger.debug(err)
1088                 if ssh:
1089                         output = ssh.close()
1090                         if verbose:
1091                                 logger.debug(output)
1092                 return errno.ETIMEDOUT
1093
1094 ### rebooting Dell systems via RAC card
1095 # Marc E. Fiuczynski - June 01 2005
1096 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1097 #
1098
1099 def runcmd(command, args, username, password, timeout = None):
1100
1101         result = [None]
1102         result_ready = threading.Condition()
1103
1104         def set_result(x):
1105
1106                 result_ready.acquire()
1107                 try:
1108                         result[0] = x
1109                 finally:
1110                         result_ready.notify()
1111                         result_ready.release()
1112
1113         def do_command(command, username, password):
1114
1115                 try:
1116                         # Popen4 is a popen-type class that combines stdout and stderr
1117                         p = popen2.Popen4(command)
1118
1119                         # read all output data
1120                         p.tochild.write("%s\n" % username)
1121                         p.tochild.write("%s\n" % password)
1122                         p.tochild.close()
1123                         data = p.fromchild.read()
1124
1125                         while True:
1126                                 # might get interrupted by a signal in poll() or waitpid()
1127                                 try:
1128                                         retval = p.wait()
1129                                         set_result((retval, data))
1130                                         break
1131                                 except OSError, ex:
1132                                         if ex.errno == errno.EINTR:
1133                                                 continue
1134                                         raise ex
1135                 except Exception, ex:
1136                         set_result(ex)
1137
1138         if args:
1139                 command = " ".join([command] + args)
1140
1141         worker = threading.Thread(target = do_command, args = (command, username, password, ))
1142         worker.setDaemon(True)
1143         result_ready.acquire()
1144         worker.start()
1145         result_ready.wait(timeout)
1146         try:
1147                 if result == [None]:
1148                         raise Exception, "command timed-out: '%s'" % command
1149         finally:
1150                 result_ready.release()
1151         result = result[0]
1152
1153         if isinstance(result, Exception):
1154                 raise result
1155         else:
1156                 (retval, data) = result
1157                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1158                         return data
1159                 else:
1160                         out = "system command ('%s') " % command
1161                         if os.WIFEXITED(retval):
1162                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1163                         else:
1164                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1165                         if data:
1166                                 out += "; output follows:\n" + data
1167                         raise Exception, out
1168
1169 def racadm_reboot(host, username, password, port, dryrun):
1170         global verbose
1171
1172         ip = socket.gethostbyname(host)
1173         try:
1174                 cmd = "/usr/sbin/racadm"
1175                 os.stat(cmd)
1176                 if not dryrun:
1177                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1178                                 username, password)
1179                 else:
1180                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1181                                 username, password)
1182
1183                 print "RUNCMD: %s" % output
1184                 if verbose:
1185                         logger.debug(output)
1186                 return 0
1187
1188         except Exception, err:
1189                 logger.debug("runcmd raised exception %s" % err)
1190                 if verbose:
1191                         logger.debug(err)
1192                 return -1
1193
1194 def pcu_name(pcu):
1195         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1196                 return pcu['hostname']
1197         elif pcu['ip'] is not None and pcu['ip'] is not "":
1198                 return pcu['ip']
1199         else:
1200                 return None
1201
1202 #import database
1203 from monitor import database
1204 fb = None
1205
1206 def get_pcu_values(pcu_id):
1207         global fb
1208         if fb == None:
1209                 # this shouldn't be loaded each time...
1210                 fb = database.dbLoad("findbadpcus")
1211
1212         try:
1213                 values = fb['nodes']["id_%s" % pcu_id]['values']
1214         except:
1215                 values = None
1216
1217         return values
1218
1219 def reboot(nodename):
1220         return reboot_policy(nodename, True, False)
1221
1222 def reboot_policy(nodename, continue_probe, dryrun):
1223         global verbose
1224
1225         pcu = plc.getpcu(nodename)
1226         if not pcu:
1227                 logger.debug("no pcu for %s" % nodename)
1228                 print "no pcu for %s" % nodename
1229                 return False # "%s has no pcu" % nodename
1230
1231         values = get_pcu_values(pcu['pcu_id'])
1232         if values == None:
1233                 logger.debug("No values for pcu probe %s" % nodename)
1234                 print "No values for pcu probe %s" % nodename
1235                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1236
1237         # Try the PCU first
1238         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1239
1240         ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1241
1242         if ret != 0:
1243                 print ret
1244                 return False
1245         else:
1246                 print "return true"
1247                 return True
1248
1249 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1250         rb_ret = ""
1251
1252         try:
1253                 # DataProbe iPal (many sites)
1254                 if  continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1255                         ipal = IPAL(values, verbose, ['23', '80', '9100'])
1256                         rb_ret = ipal.reboot(values[nodename], dryrun)
1257
1258                 # APC Masterswitch (Berkeley)
1259                 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1260                                                                   values['model'].find("Masterswitch") >= 0 ):
1261                         print values
1262
1263                         # TODO: make a more robust version of APC
1264                         if values['pcu_id'] in [1102,1163,1055,1111,1231,1113,1127,1128,1148]:
1265                                 apc = APCEurope(values, verbose, ['22', '23'])
1266                                 rb_ret = apc.reboot(values[nodename], dryrun)
1267
1268                         elif values['pcu_id'] in [1110,86]:
1269                                 apc = APCBrazil(values, verbose, ['22', '23'])
1270                                 rb_ret = apc.reboot(values[nodename], dryrun)
1271
1272                         elif values['pcu_id'] in [1221,1225,1220]:
1273                                 apc = APCBerlin(values, verbose, ['22', '23'])
1274                                 rb_ret = apc.reboot(values[nodename], dryrun)
1275
1276                         elif values['pcu_id'] in [1173,1240,47]:
1277                                 apc = APCFolsom(values, verbose, ['22', '23'])
1278                                 rb_ret = apc.reboot(values[nodename], dryrun)
1279
1280                         else:
1281                                 apc = APCMaster(values, verbose, ['22', '23'])
1282                                 rb_ret = apc.reboot(values[nodename], dryrun)
1283
1284                 # BayTech DS4-RPC
1285                 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1286                         if values['pcu_id'] in [1056,1237,1052,1209,1002,1008,1041,1013,1022]:
1287                                 # These  require a 'ctrl-c' to be sent...
1288                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1289                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1290
1291                         elif values['pcu_id'] in [93]:
1292                                 baytech = BayTechAU(values, verbose, ['22', '23'])
1293                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1294
1295                         elif values['pcu_id'] in [1057]:
1296                                 # These  require a 'ctrl-c' to be sent...
1297                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1298                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1299
1300                         elif values['pcu_id'] in [1012]:
1301                                 # This pcu sometimes doesn't present the 'Username' prompt,
1302                                 # unless you immediately try again...
1303                                 try:
1304                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1305                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1306                                 except:
1307                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1308                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1309                         else:
1310                                 baytech = BayTech(values, verbose, ['22', '23'])
1311                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1312
1313                 # iLO
1314                 elif continue_probe and values['model'].find("ilo") >= 0:
1315                         try:
1316                                 hpilo = HPiLO(values, verbose, ['22'])
1317                                 rb_ret = hpilo.reboot(0, dryrun)
1318                                 if rb_ret != 0:
1319                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1320                                         rb_ret = hpilo.reboot(0, dryrun)
1321                         except:
1322                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1323                                 rb_ret = hpilo.reboot(0, dryrun)
1324
1325                 # DRAC ssh
1326                 elif continue_probe and values['model'].find("DRAC") >= 0:
1327                         # TODO: I don't think DRACRacAdm will throw an exception for the
1328                         # default method to catch...
1329                         try:
1330                                 if values['pcu_id'] in [1402]:
1331                                         drac = DRAC(values, verbose, ['22'])
1332                                         rb_ret = drac.reboot(0, dryrun)
1333                                 else:
1334                                         drac = DRACRacAdm(values, verbose, ['443', '5869'])
1335                                         rb_ret = drac.reboot(0, dryrun)
1336                         except:
1337                                 drac = DRAC(values, verbose, ['22'])
1338                                 rb_ret = drac.reboot(0, dryrun)
1339
1340                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1341                                 wti = WTIIPS4(values, verbose, ['23'])
1342                                 rb_ret = wti.reboot(values[nodename], dryrun)
1343
1344                 elif continue_probe and values['model'].find("AMT") >= 0:
1345                                 amt = IntelAMT(values, verbose, ['16992'])
1346                                 rb_ret = amt.reboot(values[nodename], dryrun)
1347
1348                 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1349                 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1350                         # TODO: allow a different port than http 80.
1351                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1352                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1353                         elif values['pcu_id'] in [1003]:
1354                                 # OLD EPOWER
1355                                 print "OLD EPOWER"
1356                                 eps = ePowerSwitch(values, verbose, ['80'])
1357                         else:
1358                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1359
1360                         rb_ret = eps.reboot(values[nodename], dryrun)
1361                 elif continue_probe and values['pcu_id'] in [1122]:
1362                         custom = CustomPCU(values, verbose, ['80', '443'])
1363                         custom.reboot(values[nodename], dryrun)
1364
1365                 elif continue_probe:
1366                         rb_ret = "Unsupported_PCU"
1367
1368                 elif continue_probe == False:
1369                         if 'portstatus' in values:
1370                                 rb_ret = "NetDown"
1371                         else:
1372                                 rb_ret = "Not_Run"
1373                 else:
1374                         rb_ret = -1
1375
1376         except ExceptionPort, err:
1377                 rb_ret = str(err)
1378
1379         return rb_ret
1380         # ????
1381         #elif continue_probe and values['protocol'] == "racadm" and \
1382         #               values['model'] == "RAC":
1383         #       rb_ret = racadm_reboot(pcu_name(values),
1384         #                                                                 values['username'],
1385         #                                                                 values['password'],
1386         #                                                                 pcu[nodename],
1387         #                                                                 dryrun)
1388
1389 def main():
1390         logger.setLevel(logging.DEBUG)
1391         ch = logging.StreamHandler()
1392         ch.setLevel(logging.DEBUG)
1393         formatter = logging.Formatter('LOGGER - %(message)s')
1394         ch.setFormatter(formatter)
1395         logger.addHandler(ch)
1396
1397         try:
1398                 if "test" in sys.argv:
1399                         dryrun = True
1400                 else:
1401                         dryrun = False
1402
1403                 for node in sys.argv[1:]:
1404                         if node == "test": continue
1405
1406                         print "Rebooting %s" % node
1407                         if reboot_policy(node, True, dryrun):
1408                                 print "success"
1409                         else:
1410                                 print "failed"
1411         except Exception, err:
1412                 import traceback; traceback.print_exc()
1413                 from nodecommon import email_exception
1414                 email_exception()
1415                 print err
1416
1417 if __name__ == '__main__':
1418         import plc
1419         logger = logging.getLogger("monitor")
1420         main()