reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import urllib
  12 import threading, popen2
  13 import array, struct
  14 import plc
  15 import base64
  16 from subprocess import PIPE, Popen
  17 import ssh.pxssh as pxssh
  18 import ssh.pexpect as pexpect
  19 import socket
  20 import moncommands
  21
  22 # Use our versions of telnetlib and pyssh
  23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  24 import telnetlib
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  26 import pyssh
  27
  28 # Timeouts in seconds
  29 TELNET_TIMEOUT = 45
  30
  31 # Event class ID from pcu events
  32 #NODE_POWER_CONTROL = 3
  33
  34 # Monitor user ID
  35 #MONITOR_USER_ID = 11142
  36
  37 import logging
  38 logger = logging.getLogger("monitor")
  39 verbose = 1
  40 #dryrun = 0;
  41
  42 class ExceptionNoTransport(Exception): pass
  43 class ExceptionNotFound(Exception): pass
  44 class ExceptionPassword(Exception): pass
  45 class ExceptionTimeout(Exception): pass
  46 class ExceptionPrompt(Exception): pass
  47 class ExceptionSequence(Exception): pass
  48 class ExceptionReset(Exception): pass
  49 class ExceptionPort(Exception): pass
  50 class ExceptionUsername(Exception): pass
  51
  52 def telnet_answer(telnet, expected, buffer):
  53         global verbose
  54
  55         output = telnet.read_until(expected, TELNET_TIMEOUT)
  56         #if verbose:
  57         #       logger.debug(output)
  58         if output.find(expected) == -1:
  59                 raise ExceptionNotFound, "'%s' not found" % expected
  60         else:
  61                 telnet.write(buffer + "\r\n")
  62
  63
  64 # PCU has model, host, preferred-port, user, passwd,
  65
  66 # This is an object derived directly form the PLCAPI DB fields
  67 class PCU(object):
  68         def __init__(self, plc_pcu_dict):
  69                 for field in ['username', 'password', 'site_id',
  70                                                 'hostname', 'ip',
  71                                                 'pcu_id', 'model',
  72                                                 'node_ids', 'ports', ]:
  73                         if field in plc_pcu_dict:
  74                                 self.__setattr__(field, plc_pcu_dict[field])
  75                         else:
  76                                 raise Exception("No such field %s in PCU object" % field)
  77
  78 # These are the convenience functions build around the PCU object.
  79 class PCUModel(PCU):
  80         def __init__(self, plc_pcu_dict):
  81                 PCU.__init__(self, plc_pcu_dict)
  82                 self.host = self.pcu_name()
  83
  84         def pcu_name(self):
  85                 if self.hostname is not None and self.hostname is not "":
  86                         return self.hostname
  87                 elif self.ip is not None and self.ip is not "":
  88                         return self.ip
  89                 else:
  90                         return None
  91
  92         def nodeidToPort(self, node_id):
  93                 if node_id in self.node_ids:
  94                         for i in range(0, len(self.node_ids)):
  95                                 if node_id == self.node_ids[i]:
  96                                         return self.ports[i]
  97
  98                 raise Exception("No such Node ID: %d" % node_id)
  99
 100 # This class captures the observed pcu records from FindBadPCUs.py
 101 class PCURecord:
 102         def __init__(self, pcu_record_dict):
 103                 for field in ['nodenames', 'portstatus',
 104                                                 'dnsmatch',
 105                                                 'complete_entry', ]:
 106                         if field in pcu_record_dict:
 107                                 if field == "reboot":
 108                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 109                                 else:
 110                                         self.__setattr__(field, pcu_record_dict[field])
 111                         else:
 112                                 raise Exception("No such field %s in pcu record dict" % field)
 113
 114 class Transport:
 115         TELNET = 1
 116         SSH    = 2
 117         HTTP   = 3
 118         IPAL   = 4
 119
 120         TELNET_TIMEOUT = 60
 121
 122         def __init__(self, type, verbose):
 123                 self.type = type
 124                 self.verbose = verbose
 125                 self.transport = None
 126
 127         def open(self, host, username=None, password=None, prompt="User Name"):
 128                 transport = None
 129
 130                 if self.type == self.TELNET:
 131                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 132                         transport.set_debuglevel(self.verbose)
 133                         if username is not None:
 134                                 self.transport = transport
 135                                 self.ifThenSend(prompt, username, ExceptionUsername)
 136
 137                 elif self.type == self.SSH:
 138                         if username is not None:
 139                                 transport = pyssh.Ssh(username, host)
 140                                 transport.set_debuglevel(self.verbose)
 141                                 transport.open()
 142                                 # TODO: have an ssh set_debuglevel() also...
 143                         else:
 144                                 raise Exception("Username cannot be None for ssh transport.")
 145                 elif self.type == self.HTTP:
 146                         self.url = "http://%s:%d/" % (host,80)
 147                         uri = "%s:%d" % (host,80)
 148
 149                         # create authinfo
 150                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 151                         authinfo.add_password (None, uri, username, password)
 152                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 153
 154                         transport = urllib2.build_opener(authhandler)
 155
 156                 else:
 157                         raise Exception("Unknown transport type: %s" % self.type)
 158
 159                 self.transport = transport
 160                 return True
 161
 162         def close(self):
 163                 if self.type == self.TELNET:
 164                         self.transport.close()
 165                 elif self.type == self.SSH:
 166                         self.transport.close()
 167                 elif self.type == self.HTTP:
 168                         pass
 169                 else:
 170                         raise Exception("Unknown transport type %s" % self.type)
 171                 self.transport = None
 172
 173         def sendHTTP(self, resource, data):
 174                 if self.verbose:
 175                         print "POSTing '%s' to %s" % (data,self.url + resource)
 176
 177                 try:
 178                         f = self.transport.open(self.url + resource ,data)
 179                         r = f.read()
 180                         if self.verbose:
 181                                 print r
 182
 183                 except urllib2.URLError,err:
 184                         logger.info('Could not open http connection', err)
 185                         return "http transport error"
 186
 187                 return 0
 188
 189         def sendPassword(self, password, prompt=None):
 190                 if self.type == self.TELNET:
 191                         if prompt == None:
 192                                 self.ifThenSend("Password", password, ExceptionPassword)
 193                         else:
 194                                 self.ifThenSend(prompt, password, ExceptionPassword)
 195                 elif self.type == self.SSH:
 196                         self.ifThenSend("password:", password, ExceptionPassword)
 197                 elif self.type == self.HTTP:
 198                         pass
 199                 else:
 200                         raise Exception("Unknown transport type: %s" % self.type)
 201
 202         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 203
 204                 if self.transport != None:
 205                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 206                         if output.find(expected) == -1:
 207                                 raise ErrorClass, "'%s' not found" % expected
 208                         else:
 209                                 self.transport.write(buffer + "\r\n")
 210                 else:
 211                         raise ExceptionNoTransport("transport object is type None")
 212
 213         def ifElse(self, expected, ErrorClass):
 214                 try:
 215                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 216                 except:
 217                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 218
 219
 220 class PCUControl(Transport,PCUModel,PCURecord):
 221         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 222                 PCUModel.__init__(self, plc_pcu_record)
 223                 PCURecord.__init__(self, plc_pcu_record)
 224                 type = None
 225                 if self.portstatus:
 226                         if '22' in supported_ports and self.portstatus['22'] == "open":
 227                                 type = Transport.SSH
 228                         elif '23' in supported_ports and self.portstatus['23'] == "open":
 229                                 type = Transport.TELNET
 230                         elif '80' in supported_ports and self.portstatus['80'] == "open":
 231                                 type = Transport.HTTP
 232                         elif '443' in supported_ports and self.portstatus['443'] == "open":
 233                                 type = Transport.HTTP
 234                         elif '5869' in supported_ports and self.portstatus['5869'] == "open":
 235                                 # For DRAC cards. Racadm opens this port.
 236                                 type = Transport.HTTP
 237                         elif '9100' in supported_ports and self.portstatus['9100'] == "open":
 238                                 type = Transport.IPAL
 239                         elif '16992' in supported_ports and self.portstatus['16992'] == "open":
 240                                 type = Transport.HTTP
 241                         else:
 242                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 243                 else:
 244                         raise Exception("No Portstatus: No transport because no open ports")
 245                 Transport.__init__(self, type, verbose)
 246
 247         def run(self, node_port, dryrun):
 248                 """ This function is to be defined by the specific PCU instance.  """
 249                 pass
 250
 251         def reboot(self, node_port, dryrun):
 252                 try:
 253                         return self.run(node_port, dryrun)
 254                 except ExceptionNotFound, err:
 255                         return "error: " + str(err)
 256                 except ExceptionPassword, err:
 257                         return "password exception: " + str(err)
 258                 except ExceptionTimeout, err:
 259                         return "timeout exception: " + str(err)
 260                 except ExceptionUsername, err:
 261                         return "exception: no username prompt: " + str(err)
 262                 except ExceptionSequence, err:
 263                         return "sequence error: " + str(err)
 264                 except ExceptionPrompt, err:
 265                         return "prompt exception: " + str(err)
 266                 except ExceptionPort, err:
 267                         return "no ports exception: " + str(err)
 268                 except socket.error, err:
 269                         return "socket error: timeout: " + str(err)
 270                 except EOFError, err:
 271                         if self.verbose:
 272                                 logger.debug("reboot: EOF")
 273                                 logger.debug(err)
 274                         self.transport.close()
 275                         import traceback
 276                         traceback.print_exc()
 277                         return "EOF connection reset" + str(err)
 278                 except:
 279                         from nodecommon import email_exception
 280                         email_exception()
 281                         raise Exception('unknown')
 282
 283 class IPAL(PCUControl):
 284         """
 285                 This now uses a proprietary format for communicating with the PCU.  I
 286                 prefer it to Telnet, and Web access, since it's much lighter weight
 287                 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
 288         """
 289
 290         def format_msg(self, data, cmd):
 291                 esc = chr(int('1b',16))
 292                 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
 293
 294         def recv_noblock(self, s, count):
 295                 import errno
 296
 297                 try:
 298                         # TODO: make sleep backoff, before stopping.
 299                         time.sleep(8)
 300                         ret = s.recv(count, socket.MSG_DONTWAIT)
 301                 except socket.error, e:
 302                         if e[0] == errno.EAGAIN:
 303                                 #raise Exception(e[1])
 304                                 raise ExceptionNotFound(e[1])
 305                         else:
 306                                 # TODO: not other exceptions.
 307                                 raise Exception(e)
 308                 return ret
 309
 310         def run(self, node_port, dryrun):
 311                 import errno
 312
 313                 power_on = False
 314
 315                 print "open socket"
 316                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 317                 try:
 318                         print "connect"
 319                         s.connect((self.host, 9100))
 320                 except socket.error, e:
 321                         s.close()
 322                         if e[0] == errno.ECONNREFUSED:
 323                                 # cannot connect to remote host
 324                                 raise Exception(e[1])
 325                         elif e[0] == errno.ETIMEDOUT:
 326                                 raise ExceptionTimeout(e[1])
 327                         else:
 328                                 # TODO: what other conditions are there?
 329                                 raise Exception(e)
 330
 331                 # get current status
 332                 print "Checking status"
 333                 s.send(self.format_msg("", 'O'))
 334                 ret = self.recv_noblock(s, 8)
 335                 print "Current status is '%s'" % ret
 336
 337                 if ret == '':
 338                         raise Exception("Status returned 'another session already open' on %s %s : %s" % (self.host, node_port, ret))
 339
 340
 341                 if node_port < len(ret):
 342                         status = ret[node_port]
 343                         if status == '1':
 344                                 # up
 345                                 power_on = True
 346                         elif status == '0':
 347                                 # down
 348                                 power_on = False
 349                         elif status == '6':
 350                                 raise ExceptionPort("IPAL reported 'Cable Error' on %s socket %s : %s" % (self.host, node_port, ret))
 351                         else:
 352                                 raise Exception("Unknown status for PCU %s socket %s : %s" % (self.host, node_port, ret))
 353                 else:
 354                         raise Exception("Mismatch between configured port and PCU %s status: %s %s" % (self.host, node_port, ret))
 355
 356
 357                 if not dryrun:
 358                         if power_on:
 359                                 print "Pulsing %s" % node_port
 360                                 s.send(self.format_msg("%s" % node_port, 'P'))
 361                         else:
 362                                 # NOTE: turn power on ; do not pulse the port.
 363                                 print "Power was off, so turning on ..."
 364                                 s.send(self.format_msg("%s" % node_port, 'E'))
 365                                 #s.send(self.format_msg("%s" % node_port, 'P'))
 366
 367                         print "Receiving response."
 368                         ret = self.recv_noblock(s, 8)
 369                         print "Current status is '%s'" % ret
 370
 371                         if node_port < len(ret):
 372                                 status = ret[node_port]
 373                                 if status == '1':
 374                                         # up
 375                                         power_on = True
 376                                 elif status == '0':
 377                                         # down
 378                                         power_on = False
 379                                 elif status == '6':
 380                                         raise ExceptionPort("IPAL reported 'Cable Error' on %s socket %s : %s" % (self.host, node_port, ret))
 381                                 else:
 382                                         raise Exception("Unknown status for PCU %s socket %s : %s" % (self.host, node_port, ret))
 383                         else:
 384                                 raise Exception("Mismatch between configured port and PCU %s status: %s %s" % (self.host, node_port, ret))
 385
 386                         if power_on:
 387                                 return 0
 388                         else:
 389                                 return "Failed Power On"
 390
 391                 s.close()
 392                 return 0
 393
 394 # TELNET version of protocol...
 395 #               #self.open(self.host)
 396 #               ## XXX Some iPals require you to hit Enter a few times first
 397 #               #self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 398 #               # Login
 399 #               self.ifThenSend("Password >", self.password, ExceptionPassword)
 400 #               self.transport.write("\r\n\r\n")
 401 #               if not dryrun: # P# - Pulse relay
 402 #                       print "node_port %s" % node_port
 403 #                       self.ifThenSend("Enter >",
 404 #                                                       "P7", # % node_port,
 405 #                                                       ExceptionNotFound)
 406 #                       print "send newlines"
 407 #                       self.transport.write("\r\n\r\n")
 408 #                       print "after new lines"
 409 #               # Get the next prompt
 410 #               print "wait for enter"
 411 #               self.ifElse("Enter >", ExceptionTimeout)
 412 #               print "closing "
 413 #               self.close()
 414 #               return 0
 415
 416 class APCEurope(PCUControl):
 417         def run(self, node_port, dryrun):
 418                 self.open(self.host, self.username)
 419                 self.sendPassword(self.password)
 420
 421                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 422                 self.ifThenSend("\r\n> ", "2")
 423                 self.ifThenSend("\r\n> ", str(node_port))
 424                 # 3- Immediate Reboot
 425                 self.ifThenSend("\r\n> ", "3")
 426
 427                 if not dryrun:
 428                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 429                                                         "YES\r\n",
 430                                                         ExceptionSequence)
 431                 else:
 432                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 433                                                         "", ExceptionSequence)
 434                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 435
 436                 self.close()
 437                 return 0
 438
 439 class APCBrazil(PCUControl):
 440         def run(self, node_port, dryrun):
 441                 self.open(self.host, self.username)
 442                 self.sendPassword(self.password)
 443
 444                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 445                 self.ifThenSend("\r\n> ", str(node_port))
 446                 # 4- Immediate Reboot
 447                 self.ifThenSend("\r\n> ", "4")
 448
 449                 if not dryrun:
 450                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 451                                                         "YES\r\n",
 452                                                         ExceptionSequence)
 453                 else:
 454                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 455                                                         "", ExceptionSequence)
 456                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 457
 458                 self.close()
 459                 return 0
 460
 461 class APCBerlin(PCUControl):
 462         def run(self, node_port, dryrun):
 463                 self.open(self.host, self.username)
 464                 self.sendPassword(self.password)
 465
 466                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 467                 self.ifThenSend("\r\n> ", "2")
 468                 self.ifThenSend("\r\n> ", "1")
 469                 self.ifThenSend("\r\n> ", str(node_port))
 470                 # 3- Immediate Reboot
 471                 self.ifThenSend("\r\n> ", "3")
 472
 473                 if not dryrun:
 474                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 475                                                         "YES\r\n",
 476                                                         ExceptionSequence)
 477                 else:
 478                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 479                                                         "", ExceptionSequence)
 480                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 481
 482                 self.close()
 483                 return 0
 484
 485 class APCFolsom(PCUControl):
 486         def run(self, node_port, dryrun):
 487                 self.open(self.host, self.username)
 488                 self.sendPassword(self.password)
 489
 490                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 491                 self.ifThenSend("\r\n> ", "2")
 492                 self.ifThenSend("\r\n> ", "1")
 493                 self.ifThenSend("\r\n> ", str(node_port))
 494                 self.ifThenSend("\r\n> ", "1")
 495
 496                 # 3- Immediate Reboot
 497                 self.ifThenSend("\r\n> ", "3")
 498
 499                 if not dryrun:
 500                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 501                                                         "YES\r\n",
 502                                                         ExceptionSequence)
 503                 else:
 504                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 505                                                         "", ExceptionSequence)
 506                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 507
 508                 self.close()
 509                 return 0
 510
 511 class APCMaster(PCUControl):
 512         def run(self, node_port, dryrun):
 513                 print "Rebooting %s" % self.host
 514                 self.open(self.host, self.username)
 515                 self.sendPassword(self.password)
 516
 517                 # 1- Device Manager
 518                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 519                 # 3- Outlet Control/Config
 520                 self.ifThenSend("\r\n> ", "3")
 521                 # n- Outlet n
 522                 self.ifThenSend("\r\n> ", str(node_port))
 523                 # 1- Control Outlet
 524                 self.ifThenSend("\r\n> ", "1")
 525                 # 3- Immediate Reboot
 526                 self.ifThenSend("\r\n> ", "3")
 527
 528                 if not dryrun:
 529                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 530                                                         "YES\r\n",
 531                                                         ExceptionSequence)
 532                 else:
 533                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 534                                                         "", ExceptionSequence)
 535                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 536
 537                 self.close()
 538                 return 0
 539
 540 class APC(PCUControl):
 541         def __init__(self, plc_pcu_record, verbose):
 542                 PCUControl.__init__(self, plc_pcu_record, verbose)
 543
 544                 self.master = APCMaster(plc_pcu_record, verbose)
 545                 self.folsom = APCFolsom(plc_pcu_record, verbose)
 546                 self.europe = APCEurope(plc_pcu_record, verbose)
 547
 548         def run(self, node_port, dryrun):
 549                 try_again = True
 550                 sleep_time = 1
 551
 552                 for pcu in [self.master, self.europe, self.folsom]:
 553                         if try_again:
 554                                 try:
 555                                         print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
 556                                         try_again = False
 557                                         print "sleeping 5"
 558                                         time.sleep(sleep_time)
 559                                         ret = pcu.reboot(node_port, dryrun)
 560                                 except ExceptionSequence, err:
 561                                         del pcu
 562                                         sleep_time = 130
 563                                         try_again = True
 564
 565                 if try_again:
 566                         return "Unknown reboot sequence for APC PCU"
 567                 else:
 568                         return ret
 569
 570 class IntelAMT(PCUControl):
 571         def run(self, node_port, dryrun):
 572
 573                 cmd = moncommands.CMD()
 574                 #[cmd_str = "IntelAMTSDK/Samples/RemoteControl/remoteControl"
 575                 cmd_str = "cmdamt/remoteControl"
 576
 577                 if dryrun:
 578                         # NOTE: -p checks the power state of the host.
 579                         # TODO: parse the output to find out if it's ok or not.
 580                         cmd_str += " -p http://%s:16992/RemoteControlService  -user admin -pass '%s' " % (self.host, self.password )
 581                 else:
 582                         cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
 583
 584                 print cmd_str
 585                 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
 586
 587 class DRACRacAdm(PCUControl):
 588         def run(self, node_port, dryrun):
 589
 590                 print "trying racadm_reboot..."
 591                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 592
 593                 return 0
 594
 595 class DRAC(PCUControl):
 596         def run(self, node_port, dryrun):
 597                 self.open(self.host, self.username)
 598                 self.sendPassword(self.password)
 599
 600                 print "logging in..."
 601                 self.transport.write("\r\n")
 602                 # Testing Reboot ?
 603                 if dryrun:
 604                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 605                 else:
 606                         # Reset this machine
 607                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 608
 609                 self.ifThenSend("[%s]#" % self.username, "exit")
 610
 611                 self.close()
 612                 return 0
 613
 614 class HPiLO(PCUControl):
 615         def run(self, node_port, dryrun):
 616                 self.open(self.host, self.username)
 617                 self.sendPassword(self.password)
 618
 619                 # </>hpiLO->
 620                 self.ifThenSend("</>hpiLO->", "cd system1")
 621
 622                 # Reboot Outlet  N        (Y/N)?
 623                 if dryrun:
 624                         self.ifThenSend("</system1>hpiLO->", "POWER")
 625                 else:
 626                         # Reset this machine
 627                         self.ifThenSend("</system1>hpiLO->", "reset")
 628
 629                 self.ifThenSend("</system1>hpiLO->", "exit")
 630
 631                 self.close()
 632                 return 0
 633
 634
 635 class HPiLOHttps(PCUControl):
 636         def run(self, node_port, dryrun):
 637
 638                 locfg = moncommands.CMD()
 639                 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 640                                         self.host, "iloxml/Get_Network.xml",
 641                                         self.username, self.password)
 642                 sout, serr = locfg.run_noexcept(cmd)
 643
 644                 if sout.strip() != "":
 645                         print "sout: %s" % sout.strip()
 646                         return sout.strip()
 647
 648                 if not dryrun:
 649                         locfg = moncommands.CMD()
 650                         cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 651                                                 self.host, "iloxml/Reset_Server.xml",
 652                                                 self.username, self.password)
 653                         sout, serr = locfg.run_noexcept(cmd)
 654
 655                         if sout.strip() != "":
 656                                 print "sout: %s" % sout.strip()
 657                                 #return sout.strip()
 658                 return 0
 659
 660 class BayTechAU(PCUControl):
 661         def run(self, node_port, dryrun):
 662                 self.open(self.host, self.username, None, "Enter user name:")
 663                 self.sendPassword(self.password, "Enter Password:")
 664
 665                 #self.ifThenSend("RPC-16>", "Status")
 666                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 667
 668                 # Reboot Outlet  N        (Y/N)?
 669                 if dryrun:
 670                         self.ifThenSend("(Y/N)?", "N")
 671                 else:
 672                         self.ifThenSend("(Y/N)?", "Y")
 673                 self.ifThenSend("RPC3-NC>", "")
 674
 675                 self.close()
 676                 return 0
 677
 678 class BayTechGeorgeTown(PCUControl):
 679         def run(self, node_port, dryrun):
 680                 # this initial open/close is to prevent things from raising an
 681                 # exception.  the pcu always is weird during the first connection, and
 682                 # even if it's not, what does it matter to open a second connection
 683                 # right away?
 684                 self.open(self.host, self.username, None, "Enter user name:")
 685                 self.close()
 686                 time.sleep(1)
 687                 self.open(self.host, self.username, None, "Enter user name:")
 688                 self.sendPassword(self.password, "Enter Password:")
 689
 690                 #self.ifThenSend("RPC-16>", "Status")
 691
 692                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 693
 694                 # Reboot Outlet  N        (Y/N)?
 695                 if dryrun:
 696                         self.ifThenSend("(Y/N)?", "N")
 697                 else:
 698                         self.ifThenSend("(Y/N)?", "Y")
 699                 self.ifThenSend("RPC-16>", "")
 700
 701                 self.close()
 702                 return 0
 703
 704 class BayTechCtrlCUnibe(PCUControl):
 705         """
 706                 For some reason, these units let you log in fine, but they hang
 707                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 708                 why.
 709         """
 710         def run(self, node_port, dryrun):
 711                 print "BayTechCtrlC %s" % self.host
 712
 713                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 714                 s = pxssh.pxssh()
 715                 if not s.login(self.host, self.username, self.password, ssh_options):
 716                         raise ExceptionPassword("Invalid Password")
 717                 # Otherwise, the login succeeded.
 718
 719                 # Send a ctrl-c to the remote process.
 720                 print "sending ctrl-c"
 721                 s.send(chr(3))
 722
 723                 # Control Outlets  (5 ,1).........5
 724                 try:
 725                         #index = s.expect("Enter Request")
 726                         index = s.expect(["Enter Request :"])
 727
 728                         if index == 0:
 729                                 print "3"
 730                                 s.send("3\r\n")
 731                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 732                                 if index == 1:
 733                                         s.send(self.username + "\r\n")
 734                                         index = s.expect(["DS-RPC>"])
 735
 736                                 if index == 0:
 737                                         print "Reboot %d" % node_port
 738                                         s.send("Reboot %d\r\n" % node_port)
 739
 740                                         time.sleep(5)
 741                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 742                                         if index == 0:
 743                                                 if dryrun:
 744                                                         print "sending N"
 745                                                         s.send("N\r\n")
 746                                                 else:
 747                                                         print "sending Y"
 748                                                         s.send("Y\r\n")
 749                                         elif index == 1:
 750                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 751                                         elif index == 2:
 752                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 753
 754                                 time.sleep(5)
 755                                 index = s.expect(["DS-RPC>"])
 756                                 #print "got prompt back"
 757
 758                         s.close()
 759
 760                 except pexpect.EOF:
 761                         raise ExceptionPrompt("EOF before expected Prompt")
 762                 except pexpect.TIMEOUT:
 763                         raise ExceptionPrompt("Timeout before expected Prompt")
 764
 765                 return 0
 766
 767 class BayTechCtrlC(PCUControl):
 768         """
 769                 For some reason, these units let you log in fine, but they hang
 770                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 771                 why.
 772         """
 773         def run(self, node_port, dryrun):
 774                 print "BayTechCtrlC %s" % self.host
 775
 776                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 777                 s = pxssh.pxssh()
 778                 if not s.login(self.host, self.username, self.password, ssh_options):
 779                         raise ExceptionPassword("Invalid Password")
 780                 # Otherwise, the login succeeded.
 781
 782                 # Send a ctrl-c to the remote process.
 783                 print "SENDING ctrl-c"
 784                 s.send(chr(3))
 785
 786                 # Control Outlets  (5 ,1).........5
 787                 try:
 788                         print "EXPECTING: ", "Enter Request :"
 789                         index = s.expect(["Enter Request :"])
 790
 791                         if index == 0:
 792                                 print "SENDING: 5"
 793                                 s.send("5\r\n")
 794                                 print "EXPECTING: ", "DS-RPC>"
 795                                 index = s.expect(["DS-RPC>", "Enter user name:", "Port in use."])
 796                                 if index == 1:
 797                                         print "sending username"
 798                                         s.send(self.username + "\r\n")
 799                                         index = s.expect(["DS-RPC>"])
 800                                 elif index == 2:
 801                                         raise ExceptionPrompt("PCU Reported 'Port in use.'")
 802
 803                                 if index == 0:
 804                                         print "SENDING: Reboot %d" % node_port
 805                                         s.send("Reboot %d\r\n" % node_port)
 806
 807                                         print "SLEEPING: 5"
 808                                         time.sleep(5)
 809                                         print "EXPECTING: ", "Y/N?"
 810                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 811                                         if index == 0:
 812                                                 if dryrun:
 813                                                         print "sending N"
 814                                                         s.send("N\r\n")
 815                                                 else:
 816                                                         print "SENDING: Y"
 817                                                         s.send("Y\r\n")
 818                                         elif index == 1:
 819                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 820                                         elif index == 2:
 821                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 822
 823                                 # NOTE: for some reason, the script times out with the
 824                                 # following line.  In manual tests, it works correctly, but
 825                                 # with automated tests, evidently it fails.
 826                                 print "SLEEPING: 5"
 827                                 time.sleep(5)
 828                                 #print "TOTAL--", s.allstr, "--EOT"
 829                                 index = s.expect(["DS-RPC>"])
 830                                 print "got prompt back"
 831
 832                         s.close()
 833
 834                 except pexpect.EOF:
 835                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 836                 except pexpect.TIMEOUT:
 837                         raise ExceptionPrompt("Timeout before Prompt")
 838
 839                 return 0
 840
 841 class BayTech(PCUControl):
 842         def run(self, node_port, dryrun):
 843                 self.open(self.host, self.username)
 844                 self.sendPassword(self.password)
 845
 846                 # Control Outlets  (5 ,1).........5
 847                 self.ifThenSend("Enter Request :", "5")
 848
 849                 # Reboot N
 850                 try:
 851                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 852                 except ExceptionNotFound, msg:
 853                         # one machine is configured to ask for a username,
 854                         # even after login...
 855                         print "msg: %s" % msg
 856                         self.transport.write(self.username + "\r\n")
 857                         time.sleep(5)
 858                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 859
 860                 # Reboot Outlet  N        (Y/N)?
 861                 if dryrun:
 862                         self.ifThenSend("(Y/N)?", "N")
 863                 else:
 864                         self.ifThenSend("(Y/N)?", "Y")
 865                 time.sleep(5)
 866                 self.ifThenSend("DS-RPC>", "")
 867
 868                 self.close()
 869                 return 0
 870
 871 class WTIIPS4(PCUControl):
 872         def run(self, node_port, dryrun):
 873                 self.open(self.host)
 874                 self.sendPassword(self.password, "Enter Password:")
 875
 876                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 877                 if not dryrun:
 878                         self.ifThenSend("Sure? (Y/N): ", "N")
 879                 else:
 880                         self.ifThenSend("Sure? (Y/N): ", "Y")
 881
 882                 self.ifThenSend("IPS> ", "")
 883
 884                 self.close()
 885                 return 0
 886
 887 class ePowerSwitchGood(PCUControl):
 888         # NOTE:
 889         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
 890         #               For some reason this both doesn't work and in some cases, actually
 891         #               hangs the PCU.  Definitely not what we want.
 892         #
 893         #               The code below is much simpler.  Just letting things fail first,
 894         #               and then, trying again with authentication string in the header.
 895         #
 896         def run(self, node_port, dryrun):
 897                 self.transport = None
 898                 self.url = "http://%s:%d/" % (self.host,80)
 899                 uri = "%s:%d" % (self.host,80)
 900
 901                 req = urllib2.Request(self.url)
 902                 try:
 903                         handle = urllib2.urlopen(req)
 904                 except IOError, e:
 905                         # NOTE: this is expected to fail initially
 906                         pass
 907                 else:
 908                         print self.url
 909                         print "-----------"
 910                         print handle.read()
 911                         print "-----------"
 912                         return "ERROR: not protected by HTTP authentication"
 913
 914                 if not hasattr(e, 'code') or e.code != 401:
 915                         return "ERROR: failed for: %s" % str(e)
 916
 917                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
 918                 # NOTE: assuming basic realm authentication.
 919                 authheader = "Basic %s" % base64data
 920                 req.add_header("Authorization", authheader)
 921
 922                 try:
 923                         f = urllib2.urlopen(req)
 924                 except IOError, e:
 925                         # failing here means the User/passwd is wrong (hopefully)
 926                         raise ExceptionPassword("Incorrect username/password")
 927
 928                 # NOTE: after verifying that the user/password is correct,
 929                 #               actually reboot the given node.
 930                 if not dryrun:
 931                         try:
 932                                 data = urllib.urlencode({'P%d' % node_port : "r"})
 933                                 req = urllib2.Request(self.url + "cmd.html")
 934                                 req.add_header("Authorization", authheader)
 935                                 # add data to handler,
 936                                 f = urllib2.urlopen(req, data)
 937                                 if self.verbose: print f.read()
 938                         except:
 939                                 import traceback; traceback.print_exc()
 940                                 from nodecommon import email_exception
 941                                 email_exception()
 942
 943                                 # fetch url one more time on cmd.html, econtrol.html or whatever.
 944                                 # pass
 945                 else:
 946                         if self.verbose: print f.read()
 947
 948                 self.close()
 949                 return 0
 950
 951 class CustomPCU(PCUControl):
 952         def run(self, node_port, dryrun):
 953                 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
 954
 955                 if not dryrun:
 956                         # Turn host off, then on
 957                         formstr = "plab%s=off" % node_port
 958                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 959                         time.sleep(5)
 960                         formstr = "plab%s=on" % node_port
 961                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 962                 else:
 963                         os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
 964
 965
 966 class ePowerSwitchOld(PCUControl):
 967         def run(self, node_port, dryrun):
 968                 self.url = "http://%s:%d/" % (self.host,80)
 969                 uri = "%s:%d" % (self.host,80)
 970
 971                 # create authinfo
 972                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 973                 authinfo.add_password (None, uri, self.username, self.password)
 974                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 975
 976                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 977                 transport = urllib2.build_opener(authinfo)
 978                 f = transport.open(self.url)
 979                 if self.verbose: print f.read()
 980
 981                 if not dryrun:
 982                         transport = urllib2.build_opener(authhandler)
 983                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
 984                         if self.verbose: print f.read()
 985
 986                 self.close()
 987                 return 0
 988
 989 class ePowerSwitch(PCUControl):
 990         def run(self, node_port, dryrun):
 991                 self.url = "http://%s:%d/" % (self.host,80)
 992                 uri = "%s:%d" % (self.host,80)
 993
 994                 # TODO: I'm still not sure what the deal is here.
 995                 #               two independent calls appear to need to be made before the
 996                 #               reboot will succeed.  It doesn't seem to be possible to do
 997                 #               this with a single call.  I have no idea why.
 998
 999                 # create authinfo
1000                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1001                 authinfo.add_password (None, uri, self.username, self.password)
1002                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1003
1004                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
1005                 transport = urllib2.build_opener()
1006                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
1007                 if self.verbose: print f.read()
1008
1009                 if not dryrun:
1010                         transport = urllib2.build_opener(authhandler)
1011                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
1012                         if self.verbose: print f.read()
1013
1014                 #       data= "P%d=r" % node_port
1015                 #self.open(self.host, self.username, self.password)
1016                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
1017                 #self.sendHTTP("econtrol.html", data)
1018                 #self.sendHTTP("cmd.html", data)
1019
1020                 self.close()
1021                 return 0
1022
1023
1024 ### rebooting european BlackBox PSE boxes
1025 # Thierry Parmentelat - May 11 2005
1026 # tested on 4-ports models known as PSE505-FR
1027 # uses http to POST a data 'P<port>=r'
1028 # relies on basic authentication within http1.0
1029 # first curl-based script was
1030 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
1031 #       http://<hostname>:<http_port>/cmd.html && echo OK
1032
1033 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
1034
1035         global verbose
1036
1037         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
1038         data= "P%d=r" % port_in_pcu
1039         if verbose:
1040                 logger.debug("POSTing '%s' on %s" % (data,url))
1041
1042         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1043         uri = "%s:%d" % (pcu_ip,http_port)
1044         authinfo.add_password (None, uri, username, password)
1045         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1046
1047         opener = urllib2.build_opener(authhandler)
1048         urllib2.install_opener(opener)
1049
1050         if (dryrun):
1051                 return 0
1052
1053         try:
1054                 f = urllib2.urlopen(url,data)
1055
1056                 r= f.read()
1057                 if verbose:
1058                         logger.debug(r)
1059                 return 0
1060
1061         except urllib2.URLError,err:
1062                 logger.info('Could not open http connection', err)
1063                 return "bbpse error"
1064
1065 ### rebooting x10toggle based systems addressed by port
1066 # Marc E. Fiuczynski - May 31 2005
1067 # tested on 4-ports models known as PSE505-FR
1068 # uses ssh and password to login to an account
1069 # that will cause the system to be powercycled.
1070
1071 def x10toggle_reboot(ip, username, password, port, dryrun):
1072         global verbose
1073
1074         ssh = None
1075         try:
1076                 ssh = pyssh.Ssh(username, ip)
1077                 ssh.open()
1078
1079                 # Login
1080                 telnet_answer(ssh, "password:", password)
1081
1082                 if not dryrun:
1083                         # Reboot
1084                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
1085
1086                 # Close
1087                 output = ssh.close()
1088                 if verbose:
1089                         logger.debug(output)
1090                 return 0
1091
1092         except Exception, err:
1093                 if verbose:
1094                         logger.debug(err)
1095                 if ssh:
1096                         output = ssh.close()
1097                         if verbose:
1098                                 logger.debug(output)
1099                 return errno.ETIMEDOUT
1100
1101 ### rebooting Dell systems via RAC card
1102 # Marc E. Fiuczynski - June 01 2005
1103 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1104 #
1105
1106 def runcmd(command, args, username, password, timeout = None):
1107
1108         result = [None]
1109         result_ready = threading.Condition()
1110
1111         def set_result(x):
1112
1113                 result_ready.acquire()
1114                 try:
1115                         result[0] = x
1116                 finally:
1117                         result_ready.notify()
1118                         result_ready.release()
1119
1120         def do_command(command, username, password):
1121
1122                 try:
1123                         # Popen4 is a popen-type class that combines stdout and stderr
1124                         p = popen2.Popen4(command)
1125
1126                         # read all output data
1127                         p.tochild.write("%s\n" % username)
1128                         p.tochild.write("%s\n" % password)
1129                         p.tochild.close()
1130                         data = p.fromchild.read()
1131
1132                         while True:
1133                                 # might get interrupted by a signal in poll() or waitpid()
1134                                 try:
1135                                         retval = p.wait()
1136                                         set_result((retval, data))
1137                                         break
1138                                 except OSError, ex:
1139                                         if ex.errno == errno.EINTR:
1140                                                 continue
1141                                         raise ex
1142                 except Exception, ex:
1143                         set_result(ex)
1144
1145         if args:
1146                 command = " ".join([command] + args)
1147
1148         worker = threading.Thread(target = do_command, args = (command, username, password, ))
1149         worker.setDaemon(True)
1150         result_ready.acquire()
1151         worker.start()
1152         result_ready.wait(timeout)
1153         try:
1154                 if result == [None]:
1155                         raise Exception, "command timed-out: '%s'" % command
1156         finally:
1157                 result_ready.release()
1158         result = result[0]
1159
1160         if isinstance(result, Exception):
1161                 raise result
1162         else:
1163                 (retval, data) = result
1164                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1165                         return data
1166                 else:
1167                         out = "system command ('%s') " % command
1168                         if os.WIFEXITED(retval):
1169                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1170                         else:
1171                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1172                         if data:
1173                                 out += "; output follows:\n" + data
1174                         raise Exception, out
1175
1176 def racadm_reboot(host, username, password, port, dryrun):
1177         global verbose
1178
1179         ip = socket.gethostbyname(host)
1180         try:
1181                 cmd = "/usr/sbin/racadm"
1182                 os.stat(cmd)
1183                 if not dryrun:
1184                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1185                                 username, password)
1186                 else:
1187                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1188                                 username, password)
1189
1190                 print "RUNCMD: %s" % output
1191                 if verbose:
1192                         logger.debug(output)
1193                 return 0
1194
1195         except Exception, err:
1196                 logger.debug("runcmd raised exception %s" % err)
1197                 if verbose:
1198                         logger.debug(err)
1199                 return -1
1200
1201 def pcu_name(pcu):
1202         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1203                 return pcu['hostname']
1204         elif pcu['ip'] is not None and pcu['ip'] is not "":
1205                 return pcu['ip']
1206         else:
1207                 return None
1208
1209 #import database
1210 from monitor import database
1211 fb = None
1212
1213 def get_pcu_values(pcu_id):
1214         global fb
1215         if fb == None:
1216                 # this shouldn't be loaded each time...
1217                 fb = database.dbLoad("findbadpcus")
1218
1219         try:
1220                 values = fb['nodes']["id_%s" % pcu_id]['values']
1221         except:
1222                 values = None
1223
1224         return values
1225
1226 def reboot(nodename):
1227         return reboot_policy(nodename, True, False)
1228
1229 def reboot_policy(nodename, continue_probe, dryrun):
1230         global verbose
1231
1232         pcu = plc.getpcu(nodename)
1233         if not pcu:
1234                 logger.debug("no pcu for %s" % nodename)
1235                 print "no pcu for %s" % nodename
1236                 return False # "%s has no pcu" % nodename
1237
1238         values = get_pcu_values(pcu['pcu_id'])
1239         if values == None:
1240                 logger.debug("No values for pcu probe %s" % nodename)
1241                 print "No values for pcu probe %s" % nodename
1242                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1243
1244         # Try the PCU first
1245         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1246
1247         ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1248
1249         if ret != 0:
1250                 print ret
1251                 return False
1252         else:
1253                 print "return true"
1254                 return True
1255
1256 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1257         rb_ret = ""
1258
1259         try:
1260                 # DataProbe iPal (many sites)
1261                 if  continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1262                         ipal = IPAL(values, verbose, ['23', '80', '9100'])
1263                         rb_ret = ipal.reboot(values[nodename], dryrun)
1264
1265                 # APC Masterswitch (Berkeley)
1266                 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1267                                                                   values['model'].find("Masterswitch") >= 0 ):
1268                         print values
1269
1270                         # TODO: make a more robust version of APC
1271                         if values['pcu_id'] in [1102,1163,1055,1111,1231,1113,1127,1128,1148]:
1272                                 apc = APCEurope(values, verbose, ['22', '23'])
1273                                 rb_ret = apc.reboot(values[nodename], dryrun)
1274
1275                         elif values['pcu_id'] in [1110,86]:
1276                                 apc = APCBrazil(values, verbose, ['22', '23'])
1277                                 rb_ret = apc.reboot(values[nodename], dryrun)
1278
1279                         elif values['pcu_id'] in [1221,1225,1220]:
1280                                 apc = APCBerlin(values, verbose, ['22', '23'])
1281                                 rb_ret = apc.reboot(values[nodename], dryrun)
1282
1283                         elif values['pcu_id'] in [1173,1240,47]:
1284                                 apc = APCFolsom(values, verbose, ['22', '23'])
1285                                 rb_ret = apc.reboot(values[nodename], dryrun)
1286
1287                         else:
1288                                 apc = APCMaster(values, verbose, ['22', '23'])
1289                                 rb_ret = apc.reboot(values[nodename], dryrun)
1290
1291                 # BayTech DS4-RPC
1292                 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1293                         if values['pcu_id'] in [1056,1237,1052,1209,1002,1008,1041,1013,1022]:
1294                                 # These  require a 'ctrl-c' to be sent...
1295                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1296                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1297
1298                         elif values['pcu_id'] in [93]:
1299                                 baytech = BayTechAU(values, verbose, ['22', '23'])
1300                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1301
1302                         elif values['pcu_id'] in [1057]:
1303                                 # These  require a 'ctrl-c' to be sent...
1304                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1305                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1306
1307                         elif values['pcu_id'] in [1012]:
1308                                 # This pcu sometimes doesn't present the 'Username' prompt,
1309                                 # unless you immediately try again...
1310                                 try:
1311                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1312                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1313                                 except:
1314                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1315                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1316                         else:
1317                                 baytech = BayTech(values, verbose, ['22', '23'])
1318                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1319
1320                 # iLO
1321                 elif continue_probe and values['model'].find("ilo") >= 0:
1322                         try:
1323                                 hpilo = HPiLO(values, verbose, ['22'])
1324                                 rb_ret = hpilo.reboot(0, dryrun)
1325                                 if rb_ret != 0:
1326                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1327                                         rb_ret = hpilo.reboot(0, dryrun)
1328                         except:
1329                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1330                                 rb_ret = hpilo.reboot(0, dryrun)
1331
1332                 # DRAC ssh
1333                 elif continue_probe and values['model'].find("DRAC") >= 0:
1334                         # TODO: I don't think DRACRacAdm will throw an exception for the
1335                         # default method to catch...
1336                         try:
1337                                 if values['pcu_id'] in [1402]:
1338                                         drac = DRAC(values, verbose, ['22'])
1339                                         rb_ret = drac.reboot(0, dryrun)
1340                                 else:
1341                                         drac = DRACRacAdm(values, verbose, ['443', '5869'])
1342                                         rb_ret = drac.reboot(0, dryrun)
1343                         except:
1344                                 drac = DRAC(values, verbose, ['22'])
1345                                 rb_ret = drac.reboot(0, dryrun)
1346
1347                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1348                                 wti = WTIIPS4(values, verbose, ['23'])
1349                                 rb_ret = wti.reboot(values[nodename], dryrun)
1350
1351                 elif continue_probe and values['model'].find("AMT") >= 0:
1352                                 amt = IntelAMT(values, verbose, ['16992'])
1353                                 rb_ret = amt.reboot(values[nodename], dryrun)
1354
1355                 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1356                 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1357                         # TODO: allow a different port than http 80.
1358                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1359                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1360                         elif values['pcu_id'] in [1003]:
1361                                 # OLD EPOWER
1362                                 print "OLD EPOWER"
1363                                 eps = ePowerSwitch(values, verbose, ['80'])
1364                         else:
1365                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1366
1367                         rb_ret = eps.reboot(values[nodename], dryrun)
1368                 elif continue_probe and values['pcu_id'] in [1122]:
1369                         custom = CustomPCU(values, verbose, ['80', '443'])
1370                         custom.reboot(values[nodename], dryrun)
1371
1372                 elif continue_probe:
1373                         rb_ret = "Unsupported_PCU"
1374
1375                 elif continue_probe == False:
1376                         if 'portstatus' in values:
1377                                 rb_ret = "NetDown"
1378                         else:
1379                                 rb_ret = "Not_Run"
1380                 else:
1381                         rb_ret = -1
1382
1383         except ExceptionPort, err:
1384                 rb_ret = str(err)
1385
1386         return rb_ret
1387         # ????
1388         #elif continue_probe and values['protocol'] == "racadm" and \
1389         #               values['model'] == "RAC":
1390         #       rb_ret = racadm_reboot(pcu_name(values),
1391         #                                                                 values['username'],
1392         #                                                                 values['password'],
1393         #                                                                 pcu[nodename],
1394         #                                                                 dryrun)
1395
1396 def main():
1397         logger.setLevel(logging.DEBUG)
1398         ch = logging.StreamHandler()
1399         ch.setLevel(logging.DEBUG)
1400         formatter = logging.Formatter('LOGGER - %(message)s')
1401         ch.setFormatter(formatter)
1402         logger.addHandler(ch)
1403
1404         try:
1405                 if "test" in sys.argv:
1406                         dryrun = True
1407                 else:
1408                         dryrun = False
1409
1410                 for node in sys.argv[1:]:
1411                         if node == "test": continue
1412
1413                         print "Rebooting %s" % node
1414                         if reboot_policy(node, True, dryrun):
1415                                 print "success"
1416                         else:
1417                                 print "failed"
1418         except Exception, err:
1419                 import traceback; traceback.print_exc()
1420                 from nodecommon import email_exception
1421                 email_exception()
1422                 print err
1423
1424 if __name__ == '__main__':
1425         import plc
1426         logger = logging.getLogger("monitor")
1427         main()