pcucontrol/reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import urllib
  12 import threading, popen2
  13 import array, struct
  14 from monitor.wrapper import plc
  15 import base64
  16 from subprocess import PIPE, Popen
  17 import pcucontrol.transports.ssh.pxssh as pxssh
  18 import pcucontrol.transports.ssh.pexpect as pexpect
  19 import socket
  20 from monitor.util import command
  21
  22 # Use our versions of telnetlib and pyssh
  23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  24 import pcucontrol.transports.telnetlib as telnetlib
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  26 import pcucontrol.transports.pyssh as pyssh
  27
  28 # Timeouts in seconds
  29 TELNET_TIMEOUT = 45
  30
  31 # Event class ID from pcu events
  32 #NODE_POWER_CONTROL = 3
  33
  34 # Monitor user ID
  35 #MONITOR_USER_ID = 11142
  36
  37 import logging
  38 logger = logging.getLogger("monitor")
  39 verbose = 1
  40 #dryrun = 0;
  41
  42 class ExceptionNoTransport(Exception): pass
  43 class ExceptionNotFound(Exception): pass
  44 class ExceptionPassword(Exception): pass
  45 class ExceptionTimeout(Exception): pass
  46 class ExceptionPrompt(Exception): pass
  47 class ExceptionSequence(Exception): pass
  48 class ExceptionReset(Exception): pass
  49 class ExceptionPort(Exception): pass
  50 class ExceptionUsername(Exception): pass
  51
  52 def telnet_answer(telnet, expected, buffer):
  53         global verbose
  54
  55         output = telnet.read_until(expected, TELNET_TIMEOUT)
  56         #if verbose:
  57         #       logger.debug(output)
  58         if output.find(expected) == -1:
  59                 raise ExceptionNotFound, "'%s' not found" % expected
  60         else:
  61                 telnet.write(buffer + "\r\n")
  62
  63
  64 # PCU has model, host, preferred-port, user, passwd,
  65
  66 # This is an object derived directly form the PLCAPI DB fields
  67 class PCU(object):
  68         def __init__(self, plc_pcu_dict):
  69                 for field in ['username', 'password', 'site_id',
  70                                                 'hostname', 'ip',
  71                                                 'pcu_id', 'model',
  72                                                 'node_ids', 'ports', ]:
  73                         if field in plc_pcu_dict:
  74                                 self.__setattr__(field, plc_pcu_dict[field])
  75                         else:
  76                                 raise Exception("No such field %s in PCU object" % field)
  77
  78 # These are the convenience functions build around the PCU object.
  79 class PCUModel(PCU):
  80         def __init__(self, plc_pcu_dict):
  81                 PCU.__init__(self, plc_pcu_dict)
  82                 self.host = self.pcu_name()
  83
  84         def pcu_name(self):
  85                 if self.hostname is not None and self.hostname is not "":
  86                         return self.hostname
  87                 elif self.ip is not None and self.ip is not "":
  88                         return self.ip
  89                 else:
  90                         return None
  91
  92         def nodeidToPort(self, node_id):
  93                 if node_id in self.node_ids:
  94                         for i in range(0, len(self.node_ids)):
  95                                 if node_id == self.node_ids[i]:
  96                                         return self.ports[i]
  97
  98                 raise Exception("No such Node ID: %d" % node_id)
  99
 100 # This class captures the observed pcu records from FindBadPCUs.py
 101 class PCURecord:
 102         def __init__(self, pcu_record_dict):
 103                 for field in ['nodenames', 'portstatus',
 104                                                 'dnsmatch',
 105                                                 'complete_entry', ]:
 106                         if field in pcu_record_dict:
 107                                 if field == "reboot":
 108                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 109                                 else:
 110                                         self.__setattr__(field, pcu_record_dict[field])
 111                         else:
 112                                 raise Exception("No such field %s in pcu record dict" % field)
 113
 114 class Transport:
 115         TELNET = 1
 116         SSH    = 2
 117         HTTP   = 3
 118         IPAL   = 4
 119
 120         TELNET_TIMEOUT = 60
 121
 122         def __init__(self, type, verbose):
 123                 self.type = type
 124                 self.verbose = verbose
 125                 self.transport = None
 126
 127         def open(self, host, username=None, password=None, prompt="User Name"):
 128                 transport = None
 129
 130                 if self.type == self.TELNET:
 131                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 132                         transport.set_debuglevel(self.verbose)
 133                         if username is not None:
 134                                 self.transport = transport
 135                                 self.ifThenSend(prompt, username, ExceptionUsername)
 136
 137                 elif self.type == self.SSH:
 138                         if username is not None:
 139                                 transport = pyssh.Ssh(username, host)
 140                                 transport.set_debuglevel(self.verbose)
 141                                 transport.open()
 142                                 # TODO: have an ssh set_debuglevel() also...
 143                         else:
 144                                 raise Exception("Username cannot be None for ssh transport.")
 145                 elif self.type == self.HTTP:
 146                         self.url = "http://%s:%d/" % (host,80)
 147                         uri = "%s:%d" % (host,80)
 148
 149                         # create authinfo
 150                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 151                         authinfo.add_password (None, uri, username, password)
 152                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 153
 154                         transport = urllib2.build_opener(authhandler)
 155
 156                 else:
 157                         raise Exception("Unknown transport type: %s" % self.type)
 158
 159                 self.transport = transport
 160                 return True
 161
 162         def close(self):
 163                 if self.type == self.TELNET:
 164                         self.transport.close()
 165                 elif self.type == self.SSH:
 166                         self.transport.close()
 167                 elif self.type == self.HTTP:
 168                         pass
 169                 else:
 170                         raise Exception("Unknown transport type %s" % self.type)
 171                 self.transport = None
 172
 173         def sendHTTP(self, resource, data):
 174                 if self.verbose:
 175                         print "POSTing '%s' to %s" % (data,self.url + resource)
 176
 177                 try:
 178                         f = self.transport.open(self.url + resource ,data)
 179                         r = f.read()
 180                         if self.verbose:
 181                                 print r
 182
 183                 except urllib2.URLError,err:
 184                         logger.info('Could not open http connection', err)
 185                         return "http transport error"
 186
 187                 return 0
 188
 189         def sendPassword(self, password, prompt=None):
 190                 if self.type == self.TELNET:
 191                         if prompt == None:
 192                                 self.ifThenSend("Password", password, ExceptionPassword)
 193                         else:
 194                                 self.ifThenSend(prompt, password, ExceptionPassword)
 195                 elif self.type == self.SSH:
 196                         self.ifThenSend("password:", password, ExceptionPassword)
 197                 elif self.type == self.HTTP:
 198                         pass
 199                 else:
 200                         raise Exception("Unknown transport type: %s" % self.type)
 201
 202         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 203
 204                 if self.transport != None:
 205                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 206                         if output.find(expected) == -1:
 207                                 raise ErrorClass, "'%s' not found" % expected
 208                         else:
 209                                 self.transport.write(buffer + "\r\n")
 210                 else:
 211                         raise ExceptionNoTransport("transport object is type None")
 212
 213         def ifElse(self, expected, ErrorClass):
 214                 try:
 215                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 216                 except:
 217                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 218
 219
 220 class PCUControl(Transport,PCUModel,PCURecord):
 221
 222         supported_ports = []
 223
 224         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 225                 PCUModel.__init__(self, plc_pcu_record)
 226                 PCURecord.__init__(self, plc_pcu_record)
 227                 type = None
 228                 if self.portstatus:
 229                         if '22' in supported_ports and self.portstatus['22'] == "open":
 230                                 type = Transport.SSH
 231                         elif '23' in supported_ports and self.portstatus['23'] == "open":
 232                                 type = Transport.TELNET
 233                         elif '80' in supported_ports and self.portstatus['80'] == "open":
 234                                 type = Transport.HTTP
 235                         elif '443' in supported_ports and self.portstatus['443'] == "open":
 236                                 type = Transport.HTTP
 237                         elif '5869' in supported_ports and self.portstatus['5869'] == "open":
 238                                 # For DRAC cards. Racadm opens this port.
 239                                 type = Transport.HTTP
 240                         elif '9100' in supported_ports and self.portstatus['9100'] == "open":
 241                                 type = Transport.IPAL
 242                         elif '16992' in supported_ports and self.portstatus['16992'] == "open":
 243                                 type = Transport.HTTP
 244                         else:
 245                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 246                 else:
 247                         raise Exception("No Portstatus: No transport because no open ports")
 248                 Transport.__init__(self, type, verbose)
 249
 250         def run(self, node_port, dryrun):
 251                 """ This function is to be defined by the specific PCU instance.  """
 252                 pass
 253
 254         def reboot(self, node_port, dryrun):
 255                 try:
 256                         return self.run(node_port, dryrun)
 257                 except ExceptionNotFound, err:
 258                         return "error: " + str(err)
 259                 except ExceptionPassword, err:
 260                         return "password exception: " + str(err)
 261                 except ExceptionTimeout, err:
 262                         return "timeout exception: " + str(err)
 263                 except ExceptionUsername, err:
 264                         return "exception: no username prompt: " + str(err)
 265                 except ExceptionSequence, err:
 266                         return "sequence error: " + str(err)
 267                 except ExceptionPrompt, err:
 268                         return "prompt exception: " + str(err)
 269                 except ExceptionPort, err:
 270                         return "no ports exception: " + str(err)
 271                 except socket.error, err:
 272                         return "socket error: timeout: " + str(err)
 273                 except EOFError, err:
 274                         if self.verbose:
 275                                 logger.debug("reboot: EOF")
 276                                 logger.debug(err)
 277                         self.transport.close()
 278                         import traceback
 279                         traceback.print_exc()
 280                         return "EOF connection reset" + str(err)
 281
 282 class IPAL(PCUControl):
 283         """
 284                 This now uses a proprietary format for communicating with the PCU.  I
 285                 prefer it to Telnet, and Web access, since it's much lighter weight
 286                 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
 287         """
 288         supported_ports = [23,80,9100]
 289
 290         def format_msg(self, data, cmd):
 291                 esc = chr(int('1b',16))
 292                 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
 293
 294         def recv_noblock(self, s, count):
 295                 import errno
 296
 297                 try:
 298                         # TODO: make sleep backoff, before stopping.
 299                         time.sleep(4)
 300                         ret = s.recv(count, socket.MSG_DONTWAIT)
 301                 except socket.error, e:
 302                         if e[0] == errno.EAGAIN:
 303                                 raise Exception(e[1])
 304                         else:
 305                                 # TODO: not other exceptions.
 306                                 raise Exception(e)
 307                 return ret
 308
 309         def run(self, node_port, dryrun):
 310                 if self.type == Transport.IPAL:
 311                         return self.run_ipal(node_port, dryrun)
 312                 elif self.type == Transport.TELNET:
 313                         return self.run_telnet(node_port, dryrun)
 314                 else:
 315                         raise Exception("Unimplemented Transport for IPAL")
 316
 317         def run_telnet(self, node_port, dryrun):
 318                 # TELNET version of protocol...
 319                 self.open(self.host)
 320                 ## XXX Some iPals require you to hit Enter a few times first
 321                 self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 322                 # Login
 323                 self.ifThenSend("Password >", self.password, ExceptionPassword)
 324                 self.transport.write("\r\n\r\n")
 325                 if not dryrun: # P# - Pulse relay
 326                         print "node_port %s" % node_port
 327                         self.ifThenSend("Enter >",
 328                                                         "P7", # % node_port,
 329                                                         ExceptionNotFound)
 330                         print "send newlines"
 331                         self.transport.write("\r\n\r\n")
 332                         print "after new lines"
 333                 # Get the next prompt
 334                 print "wait for enter"
 335                 self.ifElse("Enter >", ExceptionTimeout)
 336                 print "closing "
 337                 self.close()
 338                 return 0
 339
 340         def run_ipal(self, node_port, dryrun):
 341                 import errno
 342
 343                 power_on = False
 344
 345                 print "open socket"
 346                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 347                 try:
 348                         print "connect"
 349                         s.connect((self.host, 9100))
 350                 except socket.error, e:
 351                         s.close()
 352                         if e[0] == errno.ECONNREFUSED:
 353                                 # cannot connect to remote host
 354                                 raise Exception(e[1])
 355                         else:
 356                                 # TODO: what other conditions are there?
 357                                 raise Exception(e)
 358
 359                 # get current status
 360                 print "Checking status"
 361                 s.send(self.format_msg("", 'O'))
 362                 ret = self.recv_noblock(s, 8)
 363                 print "Current status is '%s'" % ret
 364
 365                 if ret == '':
 366                         raise Exception("Status returned 'another session already open' %s : %s" % (node_port, ret))
 367
 368                 if node_port < len(ret):
 369                         status = ret[node_port]
 370                         if status == '1':
 371                                 # up
 372                                 power_on = True
 373                         elif status == '0':
 374                                 # down
 375                                 power_on = False
 376                         else:
 377                                 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 378                 else:
 379                         raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 380
 381
 382                 if not dryrun:
 383                         if power_on:
 384                                 print "Pulsing %s" % node_port
 385                                 s.send(self.format_msg("%s" % node_port, 'P'))
 386                         else:
 387                                 # NOTE: turn power on ; do not pulse the port.
 388                                 print "Power was off, so turning on ..."
 389                                 s.send(self.format_msg("%s" % node_port, 'E'))
 390                                 #s.send(self.format_msg("%s" % node_port, 'P'))
 391
 392                         print "Receiving response."
 393                         ret = self.recv_noblock(s, 8)
 394                         print "Current status is '%s'" % ret
 395
 396                         if node_port < len(ret):
 397                                 status = ret[node_port]
 398                                 if status == '1':
 399                                         # up
 400                                         power_on = True
 401                                 elif status == '0':
 402                                         # down
 403                                         power_on = False
 404                                 else:
 405                                         raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 406                         else:
 407                                 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 408
 409                         if power_on:
 410                                 return 0
 411                         else:
 412                                 return "Failed Power On"
 413
 414                 s.close()
 415                 return 0
 416
 417
 418 class APCEurope(PCUControl):
 419         def run(self, node_port, dryrun):
 420                 self.open(self.host, self.username)
 421                 self.sendPassword(self.password)
 422
 423                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 424                 self.ifThenSend("\r\n> ", "2")
 425                 self.ifThenSend("\r\n> ", str(node_port))
 426                 # 3- Immediate Reboot
 427                 self.ifThenSend("\r\n> ", "3")
 428
 429                 if not dryrun:
 430                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 431                                                         "YES\r\n",
 432                                                         ExceptionSequence)
 433                 else:
 434                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 435                                                         "", ExceptionSequence)
 436                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 437
 438                 self.close()
 439                 return 0
 440
 441 class APCBrazil(PCUControl):
 442         def run(self, node_port, dryrun):
 443                 self.open(self.host, self.username)
 444                 self.sendPassword(self.password)
 445
 446                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 447                 self.ifThenSend("\r\n> ", str(node_port))
 448                 # 4- Immediate Reboot
 449                 self.ifThenSend("\r\n> ", "4")
 450
 451                 if not dryrun:
 452                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 453                                                         "YES\r\n",
 454                                                         ExceptionSequence)
 455                 else:
 456                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 457                                                         "", ExceptionSequence)
 458                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 459
 460                 self.close()
 461                 return 0
 462
 463 class APCBerlin(PCUControl):
 464         def run(self, node_port, dryrun):
 465                 self.open(self.host, self.username)
 466                 self.sendPassword(self.password)
 467
 468                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 469                 self.ifThenSend("\r\n> ", "2")
 470                 self.ifThenSend("\r\n> ", "1")
 471                 self.ifThenSend("\r\n> ", str(node_port))
 472                 # 3- Immediate Reboot
 473                 self.ifThenSend("\r\n> ", "3")
 474
 475                 if not dryrun:
 476                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 477                                                         "YES\r\n",
 478                                                         ExceptionSequence)
 479                 else:
 480                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 481                                                         "", ExceptionSequence)
 482                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 483
 484                 self.close()
 485                 return 0
 486
 487 class APCFolsom(PCUControl):
 488         def run(self, node_port, dryrun):
 489                 self.open(self.host, self.username)
 490                 self.sendPassword(self.password)
 491
 492                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 493                 self.ifThenSend("\r\n> ", "2")
 494                 self.ifThenSend("\r\n> ", "1")
 495                 self.ifThenSend("\r\n> ", str(node_port))
 496                 self.ifThenSend("\r\n> ", "1")
 497
 498                 # 3- Immediate Reboot
 499                 self.ifThenSend("\r\n> ", "3")
 500
 501                 if not dryrun:
 502                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 503                                                         "YES\r\n",
 504                                                         ExceptionSequence)
 505                 else:
 506                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 507                                                         "", ExceptionSequence)
 508                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 509
 510                 self.close()
 511                 return 0
 512
 513 class APCMaster(PCUControl):
 514         supported_ports = [22,23]
 515         def run(self, node_port, dryrun):
 516                 print "Rebooting %s" % self.host
 517                 self.open(self.host, self.username)
 518                 self.sendPassword(self.password)
 519
 520                 # 1- Device Manager
 521                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 522                 # 3- Outlet Control/Config
 523                 self.ifThenSend("\r\n> ", "3")
 524                 # n- Outlet n
 525                 self.ifThenSend("\r\n> ", str(node_port))
 526                 # 1- Control Outlet
 527                 self.ifThenSend("\r\n> ", "1")
 528                 # 3- Immediate Reboot
 529                 self.ifThenSend("\r\n> ", "3")
 530
 531                 if not dryrun:
 532                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 533                                                         "YES\r\n",
 534                                                         ExceptionSequence)
 535                 else:
 536                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 537                                                         "", ExceptionSequence)
 538                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 539
 540                 self.close()
 541                 return 0
 542
 543 class APC(PCUControl):
 544         def __init__(self, plc_pcu_record, verbose):
 545                 PCUControl.__init__(self, plc_pcu_record, verbose)
 546
 547                 self.master = APCMaster(plc_pcu_record, verbose)
 548                 self.folsom = APCFolsom(plc_pcu_record, verbose)
 549                 self.europe = APCEurope(plc_pcu_record, verbose)
 550
 551         def run(self, node_port, dryrun):
 552                 try_again = True
 553                 sleep_time = 1
 554
 555                 for pcu in [self.master, self.europe, self.folsom]:
 556                         if try_again:
 557                                 try:
 558                                         print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
 559                                         try_again = False
 560                                         print "sleeping 5"
 561                                         time.sleep(sleep_time)
 562                                         ret = pcu.reboot(node_port, dryrun)
 563                                 except ExceptionSequence, err:
 564                                         del pcu
 565                                         sleep_time = 130
 566                                         try_again = True
 567
 568                 if try_again:
 569                         return "Unknown reboot sequence for APC PCU"
 570                 else:
 571                         return ret
 572
 573 class IntelAMT(PCUControl):
 574         supported_ports = [16992]
 575
 576         def run(self, node_port, dryrun):
 577
 578                 cmd = command.CMD()
 579                 # TODO: need to make this path universal; not relative to pwd.
 580                 cmd_str = "pcucontrol/models/intelamt/remoteControl"
 581
 582                 if dryrun:
 583                         # NOTE: -p checks the power state of the host.
 584                         # TODO: parse the output to find out if it's ok or not.
 585                         cmd_str += " -p http://%s:16992/RemoteControlService  -user admin -pass '%s' " % (self.host, self.password )
 586                 else:
 587                         cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
 588
 589                 print cmd_str
 590                 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
 591
 592 class DRACRacAdm(PCUControl):
 593         def run(self, node_port, dryrun):
 594
 595                 print "trying racadm_reboot..."
 596                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 597
 598                 return 0
 599
 600 class DRAC(PCUControl):
 601         supported_ports = [22,443,5869]
 602         def run(self, node_port, dryrun):
 603                 self.open(self.host, self.username)
 604                 self.sendPassword(self.password)
 605
 606                 print "logging in..."
 607                 self.transport.write("\r\n")
 608                 # Testing Reboot ?
 609                 if dryrun:
 610                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 611                 else:
 612                         # Reset this machine
 613                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 614
 615                 self.ifThenSend("[%s]#" % self.username, "exit")
 616
 617                 self.close()
 618                 return 0
 619
 620 class HPiLO(PCUControl):
 621         supported_ports = [22,443]
 622         def run(self, node_port, dryrun):
 623                 self.open(self.host, self.username)
 624                 self.sendPassword(self.password)
 625
 626                 # </>hpiLO->
 627                 self.ifThenSend("</>hpiLO->", "cd system1")
 628
 629                 # Reboot Outlet  N        (Y/N)?
 630                 if dryrun:
 631                         self.ifThenSend("</system1>hpiLO->", "POWER")
 632                 else:
 633                         # Reset this machine
 634                         self.ifThenSend("</system1>hpiLO->", "reset")
 635
 636                 self.ifThenSend("</system1>hpiLO->", "exit")
 637
 638                 self.close()
 639                 return 0
 640
 641
 642 class HPiLOHttps(PCUControl):
 643         supported_ports = [22,443]
 644         def run(self, node_port, dryrun):
 645
 646                 locfg = command.CMD()
 647                 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 648                                         self.host, "iloxml/Get_Network.xml",
 649                                         self.username, self.password)
 650                 sout, serr = locfg.run_noexcept(cmd)
 651
 652                 if sout.strip() != "":
 653                         print "sout: %s" % sout.strip()
 654                         return sout.strip()
 655
 656                 if not dryrun:
 657                         locfg = command.CMD()
 658                         cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 659                                                 self.host, "iloxml/Reset_Server.xml",
 660                                                 self.username, self.password)
 661                         sout, serr = locfg.run_noexcept(cmd)
 662
 663                         if sout.strip() != "":
 664                                 print "sout: %s" % sout.strip()
 665                                 #return sout.strip()
 666                 return 0
 667
 668 class BayTechAU(PCUControl):
 669         def run(self, node_port, dryrun):
 670                 self.open(self.host, self.username, None, "Enter user name:")
 671                 self.sendPassword(self.password, "Enter Password:")
 672
 673                 #self.ifThenSend("RPC-16>", "Status")
 674                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 675
 676                 # Reboot Outlet  N        (Y/N)?
 677                 if dryrun:
 678                         self.ifThenSend("(Y/N)?", "N")
 679                 else:
 680                         self.ifThenSend("(Y/N)?", "Y")
 681                 self.ifThenSend("RPC3-NC>", "")
 682
 683                 self.close()
 684                 return 0
 685
 686 class BayTechGeorgeTown(PCUControl):
 687         def run(self, node_port, dryrun):
 688                 self.open(self.host, self.username, None, "Enter user name:")
 689                 self.sendPassword(self.password, "Enter Password:")
 690
 691                 #self.ifThenSend("RPC-16>", "Status")
 692
 693                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 694
 695                 # Reboot Outlet  N        (Y/N)?
 696                 if dryrun:
 697                         self.ifThenSend("(Y/N)?", "N")
 698                 else:
 699                         self.ifThenSend("(Y/N)?", "Y")
 700                 self.ifThenSend("RPC-16>", "")
 701
 702                 self.close()
 703                 return 0
 704
 705 class BayTechCtrlCUnibe(PCUControl):
 706         """
 707                 For some reason, these units let you log in fine, but they hang
 708                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 709                 why.
 710         """
 711         def run(self, node_port, dryrun):
 712                 print "BayTechCtrlC %s" % self.host
 713
 714                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 715                 s = pxssh.pxssh()
 716                 if not s.login(self.host, self.username, self.password, ssh_options):
 717                         raise ExceptionPassword("Invalid Password")
 718                 # Otherwise, the login succeeded.
 719
 720                 # Send a ctrl-c to the remote process.
 721                 print "sending ctrl-c"
 722                 s.send(chr(3))
 723
 724                 # Control Outlets  (5 ,1).........5
 725                 try:
 726                         #index = s.expect("Enter Request")
 727                         index = s.expect(["Enter Request :"])
 728
 729                         if index == 0:
 730                                 print "3"
 731                                 s.send("3\r\n")
 732                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 733                                 if index == 1:
 734                                         s.send(self.username + "\r\n")
 735                                         index = s.expect(["DS-RPC>"])
 736
 737                                 if index == 0:
 738                                         print "Reboot %d" % node_port
 739                                         s.send("Reboot %d\r\n" % node_port)
 740
 741                                         time.sleep(5)
 742                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 743                                         if index == 0:
 744                                                 if dryrun:
 745                                                         print "sending N"
 746                                                         s.send("N\r\n")
 747                                                 else:
 748                                                         print "sending Y"
 749                                                         s.send("Y\r\n")
 750                                         elif index == 1:
 751                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 752                                         elif index == 2:
 753                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 754
 755                                 time.sleep(5)
 756                                 index = s.expect(["DS-RPC>"])
 757                                 #print "got prompt back"
 758
 759                         s.close()
 760
 761                 except pexpect.EOF:
 762                         raise ExceptionPrompt("EOF before expected Prompt")
 763                 except pexpect.TIMEOUT:
 764                         raise ExceptionPrompt("Timeout before expected Prompt")
 765
 766                 return 0
 767
 768 class BayTechCtrlC(PCUControl):
 769         """
 770                 For some reason, these units let you log in fine, but they hang
 771                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 772                 why.
 773         """
 774         def run(self, node_port, dryrun):
 775                 print "BayTechCtrlC %s" % self.host
 776
 777                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 778                 s = pxssh.pxssh()
 779                 if not s.login(self.host, self.username, self.password, ssh_options):
 780                         raise ExceptionPassword("Invalid Password")
 781                 # Otherwise, the login succeeded.
 782
 783                 # Send a ctrl-c to the remote process.
 784                 print "SENDING ctrl-c"
 785                 s.send(chr(3))
 786
 787                 # Control Outlets  (5 ,1).........5
 788                 try:
 789                         print "EXPECTING: ", "Enter Request :"
 790                         index = s.expect(["Enter Request :"])
 791
 792                         if index == 0:
 793                                 print "SENDING: 5"
 794                                 s.send("5\r\n")
 795                                 print "EXPECTING: ", "DS-RPC>"
 796                                 index = s.expect(["DS-RPC>", "Enter user name:", "Port in use."])
 797                                 if index == 1:
 798                                         print "sending username"
 799                                         s.send(self.username + "\r\n")
 800                                         index = s.expect(["DS-RPC>"])
 801                                 elif index == 2:
 802                                         raise ExceptionPrompt("PCU Reported 'Port in use.'")
 803
 804                                 if index == 0:
 805                                         print "SENDING: Reboot %d" % node_port
 806                                         s.send("Reboot %d\r\n" % node_port)
 807
 808                                         print "SLEEPING: 5"
 809                                         time.sleep(5)
 810                                         print "EXPECTING: ", "Y/N?"
 811                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 812                                         if index == 0:
 813                                                 if dryrun:
 814                                                         print "sending N"
 815                                                         s.send("N\r\n")
 816                                                 else:
 817                                                         print "SENDING: Y"
 818                                                         s.send("Y\r\n")
 819                                         elif index == 1:
 820                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 821                                         elif index == 2:
 822                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 823
 824                                 # NOTE: for some reason, the script times out with the
 825                                 # following line.  In manual tests, it works correctly, but
 826                                 # with automated tests, evidently it fails.
 827                                 print "SLEEPING: 5"
 828                                 time.sleep(5)
 829                                 #print "TOTAL--", s.allstr, "--EOT"
 830                                 index = s.expect(["DS-RPC>"])
 831                                 print "got prompt back"
 832
 833                         s.close()
 834
 835                 except pexpect.EOF:
 836                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 837                 except pexpect.TIMEOUT:
 838                         raise ExceptionPrompt("Timeout before Prompt")
 839
 840                 return 0
 841
 842 class BayTech(PCUControl):
 843         supported_ports = [22,23]
 844         def run(self, node_port, dryrun):
 845                 self.open(self.host, self.username)
 846                 self.sendPassword(self.password)
 847
 848                 # Control Outlets  (5 ,1).........5
 849                 self.ifThenSend("Enter Request :", "5")
 850
 851                 # Reboot N
 852                 try:
 853                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 854                 except ExceptionNotFound, msg:
 855                         # one machine is configured to ask for a username,
 856                         # even after login...
 857                         print "msg: %s" % msg
 858                         self.transport.write(self.username + "\r\n")
 859                         time.sleep(5)
 860                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 861
 862                 # Reboot Outlet  N        (Y/N)?
 863                 if dryrun:
 864                         self.ifThenSend("(Y/N)?", "N")
 865                 else:
 866                         self.ifThenSend("(Y/N)?", "Y")
 867                 time.sleep(5)
 868                 self.ifThenSend("DS-RPC>", "")
 869
 870                 self.close()
 871                 return 0
 872
 873 class WTIIPS4(PCUControl):
 874         supported_ports = [23]
 875         def run(self, node_port, dryrun):
 876                 self.open(self.host)
 877                 self.sendPassword(self.password, "Enter Password:")
 878
 879                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 880                 if not dryrun:
 881                         self.ifThenSend("Sure? (Y/N): ", "N")
 882                 else:
 883                         self.ifThenSend("Sure? (Y/N): ", "Y")
 884
 885                 self.ifThenSend("IPS> ", "")
 886
 887                 self.close()
 888                 return 0
 889
 890 class ePowerSwitchGood(PCUControl):
 891         # NOTE:
 892         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
 893         #               For some reason this both doesn't work and in some cases, actually
 894         #               hangs the PCU.  Definitely not what we want.
 895         #
 896         #               The code below is much simpler.  Just letting things fail first,
 897         #               and then, trying again with authentication string in the header.
 898         #
 899         def run(self, node_port, dryrun):
 900                 self.transport = None
 901                 self.url = "http://%s:%d/" % (self.host,80)
 902                 uri = "%s:%d" % (self.host,80)
 903
 904                 req = urllib2.Request(self.url)
 905                 try:
 906                         handle = urllib2.urlopen(req)
 907                 except IOError, e:
 908                         # NOTE: this is expected to fail initially
 909                         pass
 910                 else:
 911                         print self.url
 912                         print "-----------"
 913                         print handle.read()
 914                         print "-----------"
 915                         return "ERROR: not protected by HTTP authentication"
 916
 917                 if not hasattr(e, 'code') or e.code != 401:
 918                         return "ERROR: failed for: %s" % str(e)
 919
 920                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
 921                 # NOTE: assuming basic realm authentication.
 922                 authheader = "Basic %s" % base64data
 923                 req.add_header("Authorization", authheader)
 924
 925                 try:
 926                         f = urllib2.urlopen(req)
 927                 except IOError, e:
 928                         # failing here means the User/passwd is wrong (hopefully)
 929                         raise ExceptionPassword("Incorrect username/password")
 930
 931                 # NOTE: after verifying that the user/password is correct,
 932                 #               actually reboot the given node.
 933                 if not dryrun:
 934                         try:
 935                                 data = urllib.urlencode({'P%d' % node_port : "r"})
 936                                 req = urllib2.Request(self.url + "cmd.html")
 937                                 req.add_header("Authorization", authheader)
 938                                 # add data to handler,
 939                                 f = urllib2.urlopen(req, data)
 940                                 if self.verbose: print f.read()
 941                         except:
 942                                 import traceback; traceback.print_exc()
 943
 944                                 # fetch url one more time on cmd.html, econtrol.html or whatever.
 945                                 # pass
 946                 else:
 947                         if self.verbose: print f.read()
 948
 949                 self.close()
 950                 return 0
 951
 952 class CustomPCU(PCUControl):
 953         def run(self, node_port, dryrun):
 954                 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
 955
 956                 if not dryrun:
 957                         # Turn host off, then on
 958                         formstr = "plab%s=off" % node_port
 959                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 960                         time.sleep(5)
 961                         formstr = "plab%s=on" % node_port
 962                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
 963                 else:
 964                         os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
 965
 966
 967 class ePowerSwitchOld(PCUControl):
 968         def run(self, node_port, dryrun):
 969                 self.url = "http://%s:%d/" % (self.host,80)
 970                 uri = "%s:%d" % (self.host,80)
 971
 972                 # create authinfo
 973                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 974                 authinfo.add_password (None, uri, self.username, self.password)
 975                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 976
 977                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 978                 transport = urllib2.build_opener(authinfo)
 979                 f = transport.open(self.url)
 980                 if self.verbose: print f.read()
 981
 982                 if not dryrun:
 983                         transport = urllib2.build_opener(authhandler)
 984                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
 985                         if self.verbose: print f.read()
 986
 987                 self.close()
 988                 return 0
 989
 990 class ePowerSwitch(PCUControl):
 991         supported_ports = [80]
 992         def run(self, node_port, dryrun):
 993                 self.url = "http://%s:%d/" % (self.host,80)
 994                 uri = "%s:%d" % (self.host,80)
 995
 996                 # TODO: I'm still not sure what the deal is here.
 997                 #               two independent calls appear to need to be made before the
 998                 #               reboot will succeed.  It doesn't seem to be possible to do
 999                 #               this with a single call.  I have no idea why.
1000
1001                 # create authinfo
1002                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1003                 authinfo.add_password (None, uri, self.username, self.password)
1004                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1005
1006                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
1007                 transport = urllib2.build_opener()
1008                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
1009                 if self.verbose: print f.read()
1010
1011                 if not dryrun:
1012                         transport = urllib2.build_opener(authhandler)
1013                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
1014                         if self.verbose: print f.read()
1015
1016                 #       data= "P%d=r" % node_port
1017                 #self.open(self.host, self.username, self.password)
1018                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
1019                 #self.sendHTTP("econtrol.html", data)
1020                 #self.sendHTTP("cmd.html", data)
1021
1022                 self.close()
1023                 return 0
1024
1025
1026 ### rebooting european BlackBox PSE boxes
1027 # Thierry Parmentelat - May 11 2005
1028 # tested on 4-ports models known as PSE505-FR
1029 # uses http to POST a data 'P<port>=r'
1030 # relies on basic authentication within http1.0
1031 # first curl-based script was
1032 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
1033 #       http://<hostname>:<http_port>/cmd.html && echo OK
1034
1035 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
1036
1037         global verbose
1038
1039         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
1040         data= "P%d=r" % port_in_pcu
1041         if verbose:
1042                 logger.debug("POSTing '%s' on %s" % (data,url))
1043
1044         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1045         uri = "%s:%d" % (pcu_ip,http_port)
1046         authinfo.add_password (None, uri, username, password)
1047         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1048
1049         opener = urllib2.build_opener(authhandler)
1050         urllib2.install_opener(opener)
1051
1052         if (dryrun):
1053                 return 0
1054
1055         try:
1056                 f = urllib2.urlopen(url,data)
1057
1058                 r= f.read()
1059                 if verbose:
1060                         logger.debug(r)
1061                 return 0
1062
1063         except urllib2.URLError,err:
1064                 logger.info('Could not open http connection', err)
1065                 return "bbpse error"
1066
1067 ### rebooting x10toggle based systems addressed by port
1068 # Marc E. Fiuczynski - May 31 2005
1069 # tested on 4-ports models known as PSE505-FR
1070 # uses ssh and password to login to an account
1071 # that will cause the system to be powercycled.
1072
1073 def x10toggle_reboot(ip, username, password, port, dryrun):
1074         global verbose
1075
1076         ssh = None
1077         try:
1078                 ssh = pyssh.Ssh(username, ip)
1079                 ssh.open()
1080
1081                 # Login
1082                 telnet_answer(ssh, "password:", password)
1083
1084                 if not dryrun:
1085                         # Reboot
1086                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
1087
1088                 # Close
1089                 output = ssh.close()
1090                 if verbose:
1091                         logger.debug(output)
1092                 return 0
1093
1094         except Exception, err:
1095                 if verbose:
1096                         logger.debug(err)
1097                 if ssh:
1098                         output = ssh.close()
1099                         if verbose:
1100                                 logger.debug(output)
1101                 return errno.ETIMEDOUT
1102
1103 ### rebooting Dell systems via RAC card
1104 # Marc E. Fiuczynski - June 01 2005
1105 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1106 #
1107
1108 def runcmd(command, args, username, password, timeout = None):
1109
1110         result = [None]
1111         result_ready = threading.Condition()
1112
1113         def set_result(x):
1114
1115                 result_ready.acquire()
1116                 try:
1117                         result[0] = x
1118                 finally:
1119                         result_ready.notify()
1120                         result_ready.release()
1121
1122         def do_command(command, username, password):
1123
1124                 try:
1125                         # Popen4 is a popen-type class that combines stdout and stderr
1126                         p = popen2.Popen4(command)
1127
1128                         # read all output data
1129                         p.tochild.write("%s\n" % username)
1130                         p.tochild.write("%s\n" % password)
1131                         p.tochild.close()
1132                         data = p.fromchild.read()
1133
1134                         while True:
1135                                 # might get interrupted by a signal in poll() or waitpid()
1136                                 try:
1137                                         retval = p.wait()
1138                                         set_result((retval, data))
1139                                         break
1140                                 except OSError, ex:
1141                                         if ex.errno == errno.EINTR:
1142                                                 continue
1143                                         raise ex
1144                 except Exception, ex:
1145                         set_result(ex)
1146
1147         if args:
1148                 command = " ".join([command] + args)
1149
1150         worker = threading.Thread(target = do_command, args = (command, username, password, ))
1151         worker.setDaemon(True)
1152         result_ready.acquire()
1153         worker.start()
1154         result_ready.wait(timeout)
1155         try:
1156                 if result == [None]:
1157                         raise Exception, "command timed-out: '%s'" % command
1158         finally:
1159                 result_ready.release()
1160         result = result[0]
1161
1162         if isinstance(result, Exception):
1163                 raise result
1164         else:
1165                 (retval, data) = result
1166                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1167                         return data
1168                 else:
1169                         out = "system command ('%s') " % command
1170                         if os.WIFEXITED(retval):
1171                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1172                         else:
1173                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1174                         if data:
1175                                 out += "; output follows:\n" + data
1176                         raise Exception, out
1177
1178 def racadm_reboot(host, username, password, port, dryrun):
1179         global verbose
1180
1181         ip = socket.gethostbyname(host)
1182         try:
1183                 cmd = "/usr/sbin/racadm"
1184                 os.stat(cmd)
1185                 if not dryrun:
1186                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1187                                 username, password)
1188                 else:
1189                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1190                                 username, password)
1191
1192                 print "RUNCMD: %s" % output
1193                 if verbose:
1194                         logger.debug(output)
1195                 return 0
1196
1197         except Exception, err:
1198                 logger.debug("runcmd raised exception %s" % err)
1199                 if verbose:
1200                         logger.debug(err)
1201                 return -1
1202
1203 def pcu_name(pcu):
1204         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1205                 return pcu['hostname']
1206         elif pcu['ip'] is not None and pcu['ip'] is not "":
1207                 return pcu['ip']
1208         else:
1209                 return None
1210
1211 #import database
1212 from monitor import database
1213 fb = None
1214
1215 def get_pcu_values(pcu_id):
1216         global fb
1217         if fb == None:
1218                 # this shouldn't be loaded each time...
1219                 fb = database.dbLoad("findbadpcus")
1220
1221         try:
1222                 values = fb['nodes']["id_%s" % pcu_id]['values']
1223         except:
1224                 values = None
1225
1226         return values
1227
1228 def reboot(nodename):
1229         return reboot_policy(nodename, True, False)
1230
1231 def reboot_policy(nodename, continue_probe, dryrun):
1232         global verbose
1233
1234         pcu = plc.getpcu(nodename)
1235         if not pcu:
1236                 logger.debug("no pcu for %s" % hostname)
1237                 print "no pcu for %s" % hostname
1238                 return False # "%s has no pcu" % nodename
1239
1240         values = get_pcu_values(pcu['pcu_id'])
1241         if values == None:
1242                 logger.debug("No values for pcu probe %s" % hostname)
1243                 print "No values for pcu probe %s" % hostname
1244                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1245
1246         # Try the PCU first
1247         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1248
1249         ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1250
1251         if ret != 0:
1252                 print ret
1253                 return False
1254         else:
1255                 print "return true"
1256                 return True
1257
1258 class Unknown(PCUControl):
1259         supported_ports = [22,23,80,443,5869,9100,16992]
1260
1261 def model_to_object(modelname):
1262         if "AMT" in modelname:
1263                 return IntelAMT
1264         elif "DS4-RPC" in modelname:
1265                 return BayTech
1266         elif "ilo2" in modelname:
1267                 return HPiLO
1268         elif "IP-41x" in modelname:
1269                 return IPAL
1270         elif "AP79xx" in modelname or "Masterswitch" in modelname:
1271                 return APCMaster
1272         elif "DRAC" in modelname:
1273                 return DRAC
1274         elif "WTI" in modelname:
1275                 return WTIIPS4
1276         elif "ePowerSwitch" in modelname:
1277                 return ePowerSwitch
1278         else:
1279                 return Unknown
1280
1281 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1282         rb_ret = ""
1283         if 'plc_pcu_stats' in values:
1284                 values.update(values['plc_pcu_stats'])
1285
1286         try:
1287                 # DataProbe iPal (many sites)
1288                 if  continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1289                         ipal = IPAL(values, verbose, ['23', '80', '9100'])
1290                         rb_ret = ipal.reboot(values[nodename], dryrun)
1291
1292                 # APC Masterswitch (Berkeley)
1293                 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1294                                                                   values['model'].find("Masterswitch") >= 0 ):
1295                         print values
1296
1297                         # TODO: make a more robust version of APC
1298                         if values['pcu_id'] in [1102,1163,1055,1111,1231,1113,1127,1128,1148]:
1299                                 apc = APCEurope(values, verbose, ['22', '23'])
1300                                 rb_ret = apc.reboot(values[nodename], dryrun)
1301
1302                         elif values['pcu_id'] in [1110,86]:
1303                                 apc = APCBrazil(values, verbose, ['22', '23'])
1304                                 rb_ret = apc.reboot(values[nodename], dryrun)
1305
1306                         elif values['pcu_id'] in [1221,1225,1220]:
1307                                 apc = APCBerlin(values, verbose, ['22', '23'])
1308                                 rb_ret = apc.reboot(values[nodename], dryrun)
1309
1310                         elif values['pcu_id'] in [1173,1240,47]:
1311                                 apc = APCFolsom(values, verbose, ['22', '23'])
1312                                 rb_ret = apc.reboot(values[nodename], dryrun)
1313
1314                         else:
1315                                 apc = APCMaster(values, verbose, ['22', '23'])
1316                                 rb_ret = apc.reboot(values[nodename], dryrun)
1317
1318                 # BayTech DS4-RPC
1319                 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1320                         if values['pcu_id'] in [1056,1237,1052,1209,1002,1008,1041,1013,1022]:
1321                                 # These  require a 'ctrl-c' to be sent...
1322                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1323                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1324
1325                         elif values['pcu_id'] in [93]:
1326                                 baytech = BayTechAU(values, verbose, ['22', '23'])
1327                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1328
1329                         elif values['pcu_id'] in [1057]:
1330                                 # These  require a 'ctrl-c' to be sent...
1331                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1332                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1333
1334                         elif values['pcu_id'] in [1012]:
1335                                 # This pcu sometimes doesn't present the 'Username' prompt,
1336                                 # unless you immediately try again...
1337                                 try:
1338                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1339                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1340                                 except:
1341                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1342                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1343                         else:
1344                                 baytech = BayTech(values, verbose, ['22', '23'])
1345                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1346
1347                 # iLO
1348                 elif continue_probe and values['model'].find("ilo") >= 0:
1349                         try:
1350                                 hpilo = HPiLO(values, verbose, ['22'])
1351                                 rb_ret = hpilo.reboot(0, dryrun)
1352                                 if rb_ret != 0:
1353                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1354                                         rb_ret = hpilo.reboot(0, dryrun)
1355                         except:
1356                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1357                                 rb_ret = hpilo.reboot(0, dryrun)
1358
1359                 # DRAC ssh
1360                 elif continue_probe and values['model'].find("DRAC") >= 0:
1361                         # TODO: I don't think DRACRacAdm will throw an exception for the
1362                         # default method to catch...
1363                         try:
1364                                 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1365                                 rb_ret = drac.reboot(0, dryrun)
1366                         except:
1367                                 drac = DRAC(values, verbose, ['22'])
1368                                 rb_ret = drac.reboot(0, dryrun)
1369
1370                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1371                                 wti = WTIIPS4(values, verbose, ['23'])
1372                                 rb_ret = wti.reboot(values[nodename], dryrun)
1373
1374                 elif continue_probe and values['model'].find("AMT") >= 0:
1375                                 amt = IntelAMT(values, verbose, ['16992'])
1376                                 rb_ret = amt.reboot(values[nodename], dryrun)
1377
1378                 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1379                 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1380                         # TODO: allow a different port than http 80.
1381                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1382                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1383                         elif values['pcu_id'] in [1003]:
1384                                 # OLD EPOWER
1385                                 print "OLD EPOWER"
1386                                 eps = ePowerSwitch(values, verbose, ['80'])
1387                         else:
1388                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1389
1390                         rb_ret = eps.reboot(values[nodename], dryrun)
1391                 elif continue_probe and values['pcu_id'] in [1122]:
1392                         custom = CustomPCU(values, verbose, ['80', '443'])
1393                         custom.reboot(values[nodename], dryrun)
1394
1395                 elif continue_probe:
1396                         rb_ret = "Unsupported_PCU"
1397
1398                 elif continue_probe == False:
1399                         if 'portstatus' in values:
1400                                 rb_ret = "NetDown"
1401                         else:
1402                                 rb_ret = "Not_Run"
1403                 else:
1404                         rb_ret = -1
1405
1406         except ExceptionPort, err:
1407                 rb_ret = str(err)
1408
1409         return rb_ret
1410         # ????
1411         #elif continue_probe and values['protocol'] == "racadm" and \
1412         #               values['model'] == "RAC":
1413         #       rb_ret = racadm_reboot(pcu_name(values),
1414         #                                                                 values['username'],
1415         #                                                                 values['password'],
1416         #                                                                 pcu[nodename],
1417         #                                                                 dryrun)
1418
1419 def main():
1420         logger.setLevel(logging.DEBUG)
1421         ch = logging.StreamHandler()
1422         ch.setLevel(logging.DEBUG)
1423         formatter = logging.Formatter('LOGGER - %(message)s')
1424         ch.setFormatter(formatter)
1425         logger.addHandler(ch)
1426
1427         try:
1428                 if "test" in sys.argv:
1429                         dryrun = True
1430                 else:
1431                         dryrun = False
1432
1433                 for node in sys.argv[1:]:
1434                         if node == "test": continue
1435
1436                         print "Rebooting %s" % node
1437                         if reboot_policy(node, True, dryrun):
1438                                 print "success"
1439                         else:
1440                                 print "failed"
1441         except Exception, err:
1442                 import traceback; traceback.print_exc()
1443                 print err
1444
1445 if __name__ == '__main__':
1446         import plc
1447         logger = logging.getLogger("monitor")
1448         main()