pcucontrol/reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import urllib
  12 import threading, popen2
  13 import array, struct
  14 from monitor.wrapper import plc
  15 import base64
  16 from subprocess import PIPE, Popen
  17 import pcucontrol.transports.ssh.pxssh as pxssh
  18 import pcucontrol.transports.ssh.pexpect as pexpect
  19 import socket
  20 from monitor.util import command
  21
  22 # Use our versions of telnetlib and pyssh
  23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  24 import pcucontrol.transports.telnetlib as telnetlib
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  26 import pcucontrol.transports.pyssh as pyssh
  27 from monitor import config
  28
  29 from monitor.database.info.model import FindbadPCURecord
  30
  31 # Timeouts in seconds
  32 TELNET_TIMEOUT = 45
  33
  34 # Event class ID from pcu events
  35 #NODE_POWER_CONTROL = 3
  36
  37 # Monitor user ID
  38 #MONITOR_USER_ID = 11142
  39
  40 import logging
  41 logger = logging.getLogger("monitor")
  42 verbose = 1
  43 #dryrun = 0;
  44
  45 class ExceptionNoTransport(Exception): pass
  46 class ExceptionNotFound(Exception): pass
  47 class ExceptionPassword(Exception): pass
  48 class ExceptionTimeout(Exception): pass
  49 class ExceptionPrompt(Exception): pass
  50 class ExceptionSequence(Exception): pass
  51 class ExceptionReset(Exception): pass
  52 class ExceptionPort(Exception): pass
  53 class ExceptionUsername(Exception): pass
  54
  55 def telnet_answer(telnet, expected, buffer):
  56         global verbose
  57
  58         output = telnet.read_until(expected, TELNET_TIMEOUT)
  59         #if verbose:
  60         #       logger.debug(output)
  61         if output.find(expected) == -1:
  62                 raise ExceptionNotFound, "'%s' not found" % expected
  63         else:
  64                 telnet.write(buffer + "\r\n")
  65
  66
  67 # PCU has model, host, preferred-port, user, passwd,
  68
  69 # This is an object derived directly form the PLCAPI DB fields
  70 class PCU(object):
  71         def __init__(self, plc_pcu_dict):
  72                 for field in ['username', 'password', 'site_id',
  73                                                 'hostname', 'ip',
  74                                                 'pcu_id', 'model',
  75                                                 'node_ids', 'ports', ]:
  76                         if field in plc_pcu_dict:
  77                                 self.__setattr__(field, plc_pcu_dict[field])
  78                         else:
  79                                 raise Exception("No such field %s in PCU object" % field)
  80
  81 # These are the convenience functions build around the PCU object.
  82 class PCUModel(PCU):
  83         def __init__(self, plc_pcu_dict):
  84                 PCU.__init__(self, plc_pcu_dict)
  85                 self.host = self.pcu_name()
  86
  87         def pcu_name(self):
  88                 if self.hostname is not None and self.hostname is not "":
  89                         return self.hostname
  90                 elif self.ip is not None and self.ip is not "":
  91                         return self.ip
  92                 else:
  93                         return None
  94
  95         def nodeidToPort(self, node_id):
  96                 if node_id in self.node_ids:
  97                         for i in range(0, len(self.node_ids)):
  98                                 if node_id == self.node_ids[i]:
  99                                         return self.ports[i]
 100
 101                 raise Exception("No such Node ID: %d" % node_id)
 102
 103 # This class captures the observed pcu records from FindBadPCUs.py
 104 class PCURecord:
 105         def __init__(self, pcu_record_dict):
 106                 for field in ['port_status',
 107                                                 'dns_status',
 108                                                 'entry_complete', ]:
 109                         if field in pcu_record_dict:
 110                                 if field == "reboot":
 111                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 112                                 else:
 113                                         self.__setattr__(field, pcu_record_dict[field])
 114                         else:
 115                                 raise Exception("No such field %s in pcu record dict" % field)
 116
 117 class Transport:
 118         TELNET = 1
 119         SSH    = 2
 120         HTTP   = 3
 121         HTTPS  = 4
 122         IPAL   = 5
 123         DRAC   = 6
 124
 125         TELNET_TIMEOUT = 120
 126
 127         def __init__(self, type, verbose):
 128                 self.type = type
 129                 self.verbose = verbose
 130                 self.transport = None
 131
 132         def open(self, host, username=None, password=None, prompt="User Name"):
 133                 transport = None
 134
 135                 if self.type == self.TELNET:
 136                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 137                         transport.set_debuglevel(self.verbose)
 138                         if username is not None:
 139                                 self.transport = transport
 140                                 self.ifThenSend(prompt, username, ExceptionUsername)
 141
 142                 elif self.type == self.SSH:
 143                         if username is not None:
 144                                 transport = pyssh.Ssh(username, host)
 145                                 transport.set_debuglevel(self.verbose)
 146                                 transport.open()
 147                                 # TODO: have an ssh set_debuglevel() also...
 148                         else:
 149                                 raise Exception("Username cannot be None for ssh transport.")
 150                 elif self.type == self.HTTP:
 151                         self.url = "http://%s:%d/" % (host,80)
 152                         uri = "%s:%d" % (host,80)
 153
 154                         # create authinfo
 155                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 156                         authinfo.add_password (None, uri, username, password)
 157                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 158
 159                         transport = urllib2.build_opener(authhandler)
 160
 161                 else:
 162                         raise Exception("Unknown transport type: %s" % self.type)
 163
 164                 self.transport = transport
 165                 return True
 166
 167         def close(self):
 168                 if self.type == self.TELNET:
 169                         self.transport.close()
 170                 elif self.type == self.SSH:
 171                         self.transport.close()
 172                 elif self.type == self.HTTP:
 173                         pass
 174                 else:
 175                         raise Exception("Unknown transport type %s" % self.type)
 176                 self.transport = None
 177
 178         def sendHTTP(self, resource, data):
 179                 if self.verbose:
 180                         print "POSTing '%s' to %s" % (data,self.url + resource)
 181
 182                 try:
 183                         f = self.transport.open(self.url + resource ,data)
 184                         r = f.read()
 185                         if self.verbose:
 186                                 print r
 187
 188                 except urllib2.URLError,err:
 189                         logger.info('Could not open http connection', err)
 190                         return "http transport error"
 191
 192                 return 0
 193
 194         def sendPassword(self, password, prompt=None):
 195                 if self.type == self.TELNET:
 196                         if prompt == None:
 197                                 self.ifThenSend("Password", password, ExceptionPassword)
 198                         else:
 199                                 self.ifThenSend(prompt, password, ExceptionPassword)
 200                 elif self.type == self.SSH:
 201                         self.ifThenSend("password:", password, ExceptionPassword)
 202                 elif self.type == self.HTTP:
 203                         pass
 204                 else:
 205                         raise Exception("Unknown transport type: %s" % self.type)
 206
 207         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 208
 209                 if self.transport != None:
 210                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 211                         if output.find(expected) == -1:
 212                                 print "OUTPUT: --%s--" % output
 213                                 raise ErrorClass, "'%s' not found" % expected
 214                         else:
 215                                 self.transport.write(buffer + "\r\n")
 216                 else:
 217                         raise ExceptionNoTransport("transport object is type None")
 218
 219         def ifElse(self, expected, ErrorClass):
 220                 try:
 221                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 222                 except:
 223                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 224
 225
 226 class PCUControl(Transport,PCUModel,PCURecord):
 227
 228         supported_ports = []
 229
 230         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 231                 PCUModel.__init__(self, plc_pcu_record)
 232                 PCURecord.__init__(self, plc_pcu_record)
 233                 type = None
 234                 if self.port_status:
 235                         # NOTE: prefer racadm port over ssh
 236                         if '5869' in supported_ports and self.port_status['5869'] == "open":
 237                                 type = Transport.DRAC# DRAC cards user this port.
 238                         elif '22' in supported_ports and self.port_status['22'] == "open":
 239                                 type = Transport.SSH
 240                         elif '23' in supported_ports and self.port_status['23'] == "open":
 241                                 type = Transport.TELNET
 242                         # NOTE: prefer https over http
 243                         elif '443' in supported_ports and self.port_status['443'] == "open":
 244                                 type = Transport.HTTPS
 245                         elif '80' in supported_ports and self.port_status['80'] == "open":
 246                                 type = Transport.HTTP
 247                         elif '9100' in supported_ports and self.port_status['9100'] == "open":
 248                                 type = Transport.IPAL
 249                         elif '16992' in supported_ports and self.port_status['16992'] == "open":
 250                                 type = Transport.HTTP
 251                         else:
 252                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 253                 else:
 254                         raise ExceptionPort("No Portstatus: No transport because no open ports")
 255                 Transport.__init__(self, type, verbose)
 256
 257         def run(self, node_port, dryrun):
 258                 """ This function is to be defined by the specific PCU instance.  """
 259                 pass
 260
 261         def reboot(self, node_port, dryrun):
 262                 try:
 263                         return self.run(node_port, dryrun)
 264                 except ExceptionNotFound, err:
 265                         return "error: " + str(err)
 266                 except ExceptionPassword, err:
 267                         return "Password exception: " + str(err)
 268                 except ExceptionTimeout, err:
 269                         return "Timeout exception: " + str(err)
 270                 except ExceptionUsername, err:
 271                         return "No username prompt: " + str(err)
 272                 except ExceptionSequence, err:
 273                         return "Sequence error: " + str(err)
 274                 except ExceptionPrompt, err:
 275                         return "Prompt exception: " + str(err)
 276                 except ExceptionNoTransport, err:
 277                         return "No Transport: " + str(err)
 278                 except ExceptionPort, err:
 279                         return "No ports exception: " + str(err)
 280                 except socket.error, err:
 281                         return "socket error: timeout: " + str(err)
 282                 except urllib2.HTTPError, err:
 283                         return "HTTPError: " + str(err)
 284                 except urllib2.URLError, err:
 285                         return "URLError: " + str(err)
 286                 except EOFError, err:
 287                         if self.verbose:
 288                                 logger.debug("reboot: EOF")
 289                                 logger.debug(err)
 290                         self.transport.close()
 291                         import traceback
 292                         traceback.print_exc()
 293                         return "EOF connection reset" + str(err)
 294
 295 class IPMI(PCUControl):
 296
 297         supported_ports = [80,443,623]
 298
 299         # TODO: get exit codes to determine success or failure...
 300         def run(self, node_port, dryrun):
 301
 302                 if not dryrun:
 303                         cmd = "ipmitool -I lanplus -H %s -U %s -P '%s' power cycle"
 304                         p = os.popen(cmd % ( self.host, self.username, self.password) )
 305                         result = p.read()
 306                         print "RESULT: ", result
 307                 else:
 308                         cmd = "ipmitool -I lanplus -H %s -U %s -P '%s' user list"
 309                         p = os.popen(cmd % ( self.host, self.username, self.password) )
 310                         result = p.read()
 311                         print "RESULT: ", result
 312
 313                 if "Error" in result:
 314                         return result
 315                 else:
 316                         return 0
 317
 318 class IPAL(PCUControl):
 319         """
 320                 This now uses a proprietary format for communicating with the PCU.  I
 321                 prefer it to Telnet, and Web access, since it's much lighter weight
 322                 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
 323         """
 324         supported_ports = [23,80,9100]
 325
 326         def format_msg(self, data, cmd):
 327                 esc = chr(int('1b',16))
 328                 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
 329
 330         def recv_noblock(self, s, count):
 331                 import errno
 332
 333                 try:
 334                         # TODO: make sleep backoff, before stopping.
 335                         time.sleep(4)
 336                         ret = s.recv(count, socket.MSG_DONTWAIT)
 337                 except socket.error, e:
 338                         if e[0] == errno.EAGAIN:
 339                                 raise Exception(e[1])
 340                         else:
 341                                 # TODO: not other exceptions.
 342                                 raise Exception(e)
 343                 return ret
 344
 345         def run(self, node_port, dryrun):
 346                 if self.type == Transport.IPAL:
 347                         ret = self.run_ipal(node_port, dryrun)
 348                         if ret != 0:
 349                                 ret2 = self.run_telnet(node_port, dryrun)
 350                                 if ret2 != 0:
 351                                         return ret
 352                                 return ret2
 353                         return ret
 354                 elif self.type == Transport.TELNET:
 355                         return self.run_telnet(node_port, dryrun)
 356                 else:
 357                         raise ExceptionNoTransport("Unimplemented Transport for IPAL")
 358
 359         def run_telnet(self, node_port, dryrun):
 360                 # TELNET version of protocol...
 361                 self.open(self.host)
 362                 ## XXX Some iPals require you to hit Enter a few times first
 363                 self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 364                 # Login
 365                 self.ifThenSend("Password >", self.password, ExceptionPassword)
 366                 self.transport.write("\r\n\r\n")
 367                 if not dryrun: # P# - Pulse relay
 368                         print "node_port %s" % node_port
 369                         self.ifThenSend("Enter >",
 370                                                         "P7", # % node_port,
 371                                                         ExceptionNotFound)
 372                         print "send newlines"
 373                         self.transport.write("\r\n\r\n")
 374                         print "after new lines"
 375                 # Get the next prompt
 376                 print "wait for enter"
 377                 self.ifElse("Enter >", ExceptionTimeout)
 378                 print "closing "
 379                 self.close()
 380                 return 0
 381
 382         def run_ipal(self, node_port, dryrun):
 383                 import errno
 384
 385                 power_on = False
 386
 387                 print "open socket"
 388                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 389                 try:
 390                         print "connect"
 391                         s.connect((self.host, 9100))
 392                 except socket.error, e:
 393                         s.close()
 394                         if e[0] == errno.ECONNREFUSED:
 395                                 # cannot connect to remote host
 396                                 raise Exception(e[1])
 397                         else:
 398                                 # TODO: what other conditions are there?
 399                                 raise Exception(e)
 400
 401                 # get current status
 402                 print "Checking status"
 403                 s.send(self.format_msg("", 'O'))
 404                 ret = self.recv_noblock(s, 8)
 405                 print "Current status is '%s'" % ret
 406
 407                 if ret == '':
 408                         raise Exception("Status returned 'another session already open' %s : %s" % (node_port, ret))
 409
 410                 if node_port < len(ret):
 411                         status = ret[node_port]
 412                         if status == '1':
 413                                 # up
 414                                 power_on = True
 415                         elif status == '0':
 416                                 # down
 417                                 power_on = False
 418                         else:
 419                                 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 420                 else:
 421                         raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 422
 423
 424                 if not dryrun:
 425                         if power_on:
 426                                 print "Pulsing %s" % node_port
 427                                 s.send(self.format_msg("%s" % node_port, 'P'))
 428                         else:
 429                                 # NOTE: turn power on ; do not pulse the port.
 430                                 print "Power was off, so turning on ..."
 431                                 s.send(self.format_msg("%s" % node_port, 'E'))
 432                                 #s.send(self.format_msg("%s" % node_port, 'P'))
 433
 434                         print "Receiving response."
 435                         ret = self.recv_noblock(s, 8)
 436                         print "Current status is '%s'" % ret
 437
 438                         if node_port < len(ret):
 439                                 status = ret[node_port]
 440                                 if status == '1':
 441                                         # up
 442                                         power_on = True
 443                                 elif status == '0':
 444                                         # down
 445                                         power_on = False
 446                                 else:
 447                                         raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 448                         else:
 449                                 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 450
 451                         if power_on:
 452                                 return 0
 453                         else:
 454                                 return "Failed Power On"
 455
 456                 s.close()
 457                 return 0
 458
 459 class APCControl(PCUControl):
 460         supported_ports = [22,23,80,443]
 461         reboot_sequence = []
 462
 463         def run(self, node_port, dryrun):
 464                 print "RUNNING!!!!!!!!!!!!"
 465                 if self.type == Transport.HTTPS or self.type == Transport.HTTP:
 466                         print "APC via http...."
 467                         return self.run_http_or_https(node_port, dryrun)
 468                 else:
 469                         print "APC via telnet/ssh...."
 470                         return self.run_telnet_or_ssh(node_port, dryrun)
 471
 472         def run_telnet_or_ssh(self, node_port, dryrun):
 473                 self.open(self.host, self.username)
 474                 self.sendPassword(self.password)
 475
 476                 first = True
 477                 for val in self.reboot_sequence:
 478                         if first:
 479                                 self.ifThenSend("\r\n> ", val, ExceptionPassword)
 480                                 first = False
 481                         else:
 482                                 self.ifThenSend("\r\n> ", val)
 483
 484                 if not dryrun:
 485                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 486                                                         "YES\r\n",
 487                                                         ExceptionSequence)
 488                 else:
 489                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 490                                                         "", ExceptionSequence)
 491                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 492
 493                 self.close()
 494                 return 0
 495
 496         def run_http_or_https(self, node_port, dryrun):
 497                 if not dryrun:
 498                         # send reboot signal.
 499                         # TODO: send a ManualPCU() reboot request for this PCU.
 500                         # NOTE: this model defies automation because, the port numbering
 501                         #       and the form numbers are not consistent across models.  There is
 502                         #       not direct mapping from port# to form#.
 503                         return "Manual Reboot Required"
 504
 505                 else:
 506                         # TODO: also send message for https, since that doesn't work this way...
 507                         if self.type == Transport.HTTPS:
 508                                 cmd = self.get_https_cmd()
 509                         elif self.type == Transport.HTTP:
 510                                 cmd = self.get_http_cmd()
 511                         else:
 512                                 raise ExceptionNoTransport("Unsupported transport for http command")
 513
 514                 cmd = cmd % ( self.username, self.password, self.host)
 515                 print "CMD: %s" % cmd
 516
 517                 p = os.popen(cmd)
 518                 result = p.read()
 519                 if len(result.split('\n')) > 2:
 520                         self.logout()
 521                         return 0
 522                 else:
 523                         # NOTE: an error has occurred, so no need to log out.
 524                         print "RESULT: ", result
 525                         return result
 526
 527         def get_https_cmd(self):
 528                 version = self.get_version()
 529                 print "VERSION: %s" % version
 530                 if "AP96" in version:
 531                         cmd = "curl -s --insecure --user '%s:%s' https://%s/outlets.htm " + \
 532                                   " | grep -E '^[^<]+' " + \
 533                                   " | grep -v 'Protected Object' "
 534                 else:
 535                         # NOTE: no other case known right now...
 536                         cmd = "curl -s --insecure --user '%s:%s' https://%s/outlets.htm " + \
 537                                   " | grep -E '^[^<]+' " + \
 538                                   " | grep -v 'Protected Object' "
 539
 540                 return cmd
 541
 542         def get_http_cmd(self):
 543                 version = self.get_version()
 544                 print "VERSION: %s" % version
 545                 if "AP7900" in version:
 546                         cmd = "curl -s --anyauth --user '%s:%s' http://%s/rPDUout.htm | grep -E '^[^<]+'"
 547                 elif "AP7920" in version:
 548                         cmd = "curl -s --anyauth --user '%s:%s' http://%s/ms3out.htm | grep -E '^[^<]+' "
 549                 else:
 550                         # default case...
 551                         print "USING DEFAULT"
 552                         cmd = "curl -s --anyauth --user '%s:%s' http://%s/ms3out.htm | grep -E '^[^<]+' "
 553
 554                 return cmd
 555
 556         def get_version(self):
 557                 # NOTE: this command returns and formats all data.
 558                 #cmd = """curl -s --anyauth --user '%s:%s' http://%s/about.htm """ +
 559                 #      """ | sed -e "s/<[^>]*>//g" -e "s/&nbsp;//g" -e "/^$/d" """ +
 560                 #         """ | awk '{line=$0 ; if ( ! /:/ && length(pline) > 0 ) \
 561                 #                            { print pline, line } else { pline=line} }' """ +
 562                 #         """ | grep Model """
 563
 564                 # NOTE: we may need to return software version, no model version to
 565                 #               know which file to request on the server.
 566
 567                 if self.type == Transport.HTTP:
 568                         cmd = """curl -s --anyauth --user '%s:%s' http://%s/about.htm """ + \
 569                                   """ | sed -e "s/<[^>]*>//g" -e "s/&nbsp;//g" -e "/^$/d" """ + \
 570                                   """ | grep -E "AP[[:digit:]]+" """
 571                                   #""" | grep -E "v[[:digit:]].*" """
 572                 elif self.type == Transport.HTTPS:
 573                         cmd = """curl -s --insecure --user '%s:%s' https://%s/about.htm """ + \
 574                                   """ | sed -e "s/<[^>]*>//g" -e "s/&nbsp;//g" -e "/^$/d" """ + \
 575                                   """ | grep -E "AP[[:digit:]]+" """
 576                                   #""" | grep -E "v[[:digit:]].*" """
 577                 else:
 578                         raise ExceptionNoTransport("Unsupported transport to get version")
 579
 580                 cmd = cmd % ( self.username, self.password, self.host)
 581                 p = os.popen(cmd)
 582                 result = p.read()
 583                 return result.strip()
 584
 585         def logout(self):
 586                 # NOTE: log out again, to allow other uses to access the machine.
 587                 if self.type == Transport.HTTP:
 588                         cmd = """curl -s --anyauth --user '%s:%s' http://%s/logout.htm """ + \
 589                                   """ | grep -E '^[^<]+' """
 590                 elif self.type == Transport.HTTPS:
 591                         cmd = """curl -s --insecure --user '%s:%s' http://%s/logout.htm """ + \
 592                                   """ | grep -E '^[^<]+' """
 593                 else:
 594                         raise ExceptionNoTransport("Unsupported transport to logout")
 595
 596                 cmd = cmd % ( self.username, self.password, self.host)
 597                 p = os.popen(cmd)
 598                 print p.read()
 599
 600 class APCControl12p3(APCControl):
 601         def run(self, node_port, dryrun):
 602                 self.reboot_sequence = ["1", "2", str(node_port), "3"]
 603                 return super(APCControl12p3, self).run(node_port, dryrun)
 604
 605 class APCControl1p4(APCControl):
 606         def run(self, node_port, dryrun):
 607                 self.reboot_sequence = ["1", str(node_port), "4"]
 608                 return super(APCControl1p4, self).run(node_port, dryrun)
 609
 610 class APCControl121p3(APCControl):
 611         def run(self, node_port, dryrun):
 612                 self.reboot_sequence = ["1", "2", "1", str(node_port), "3"]
 613                 return super(APCControl121p3, self).run(node_port, dryrun)
 614
 615 class APCControl121p1(APCControl):
 616         def run(self, node_port, dryrun):
 617                 self.reboot_sequence = ["1", "2", "1", str(node_port), "1", "3"]
 618                 return super(APCControl121p1, self).run(node_port, dryrun)
 619
 620 class APCControl13p13(APCControl):
 621         def run(self, node_port, dryrun):
 622                 self.reboot_sequence = ["1", "3", str(node_port), "1", "3"]
 623                 return super(APCControl13p13, self).run(node_port, dryrun)
 624
 625
 626 class IntelAMT(PCUControl):
 627         supported_ports = [16992]
 628
 629         def run(self, node_port, dryrun):
 630
 631                 cmd = command.CMD()
 632                 # TODO: need to make this path universal; not relative to pwd.
 633                 cmd_str = config.MONITOR_SCRIPT_ROOT + "/pcucontrol/models/intelamt/remoteControl"
 634
 635                 if dryrun:
 636                         # NOTE: -p checks the power state of the host.
 637                         # TODO: parse the output to find out if it's ok or not.
 638                         cmd_str += " -p http://%s:16992/RemoteControlService  -user admin -pass '%s' " % (self.host, self.password )
 639                 else:
 640                         cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
 641
 642                 print cmd_str
 643                 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
 644
 645 class DRAC(PCUControl):
 646         supported_ports = [22,443,5869]
 647         def run(self, node_port, dryrun):
 648                 if self.type == Transport.DRAC:
 649                         print "trying racadm_reboot..."
 650                         return racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 651                 elif self.type == Transport.SSH:
 652                         return self.run_ssh(node_port, dryrun)
 653                 else:
 654                         raise ExceptionNoTransport("No implementation for open ports")
 655
 656         def run_ssh(self, node_port, dryrun):
 657                 ssh_options="-o StrictHostKeyChecking=no "+\
 658                             "-o PasswordAuthentication=yes "+\
 659                                         "-o PubkeyAuthentication=no"
 660                 s = pxssh.pxssh()
 661                 if not s.login(self.host, self.username, self.password, ssh_options,
 662                                                 original_prompts="Dell", login_timeout=TELNET_TIMEOUT):
 663                         raise ExceptionPassword("Invalid Password")
 664
 665                 print "logging in..."
 666                 s.send("\r\n\r\n")
 667                 try:
 668                         # Testing Reboot ?
 669                         #index = s.expect(["DRAC 5", "[%s]#" % self.username ])
 670                         # NOTE: be careful to escape any characters used by 're.compile'
 671                         index = s.expect(["\$", "\[%s\]#" % self.username ])
 672                         print "INDEX:", index
 673                         if dryrun:
 674                                 if index == 0:
 675                                         s.send("racadm getsysinfo")
 676                                 elif index == 1:
 677                                         s.send("getsysinfo")
 678                         else:
 679                                 if index == 0:
 680                                         s.send("racadm serveraction powercycle")
 681                                 elif index == 1:
 682                                         s.send("serveraction powercycle")
 683
 684                         s.send("exit")
 685
 686                 except pexpect.EOF:
 687                         raise ExceptionPrompt("EOF before expected Prompt")
 688                 except pexpect.TIMEOUT:
 689                         print s
 690                         raise ExceptionPrompt("Timeout before expected Prompt")
 691
 692                 s.close()
 693
 694                 return 0
 695
 696 class DRACDefault(PCUControl):
 697         supported_ports = [22,443,5869]
 698         def run(self, node_port, dryrun):
 699                 self.open(self.host, self.username)
 700                 self.sendPassword(self.password)
 701
 702                 print "logging in..."
 703                 self.transport.write("\r\n")
 704                 # Testing Reboot ?
 705                 if dryrun:
 706                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 707                 else:
 708                         # Reset this machine
 709                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 710
 711                 self.ifThenSend("[%s]#" % self.username, "exit")
 712
 713                 self.close()
 714                 return 0
 715
 716 class HPiLO(PCUControl):
 717         supported_ports = [22,443]
 718         def run(self, node_port, dryrun):
 719                 if self.type == Transport.SSH:
 720                         return self.run_ssh(node_port, dryrun)
 721                 elif self.type == Transport.HTTP or self.type == Transport.HTTPS:
 722                         return self.run_https(node_port, dryrun)
 723                 else:
 724                         raise ExceptionNoTransport("Unimplemented Transport for HPiLO %s" % self.type)
 725
 726         def run_ssh(self, node_port, dryrun):
 727
 728                 self.open(self.host, self.username)
 729                 self.sendPassword(self.password)
 730
 731                 # </>hpiLO->
 732                 self.ifThenSend("</>hpiLO->", "cd system1")
 733
 734                 # Reboot Outlet  N        (Y/N)?
 735                 if dryrun:
 736                         self.ifThenSend("</system1>hpiLO->", "POWER")
 737                 else:
 738                         # Reset this machine
 739                         self.ifThenSend("</system1>hpiLO->", "reset")
 740
 741                 self.ifThenSend("</system1>hpiLO->", "exit")
 742
 743                 self.close()
 744                 return 0
 745
 746         def run_https(self, node_port, dryrun):
 747
 748                 locfg = command.CMD()
 749
 750                 cmd_str = config.MONITOR_SCRIPT_ROOT + "/pcucontrol/models/hpilo/"
 751
 752                 cmd = cmd_str + "locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 753                                         self.host, cmd_str+"iloxml/Get_Network.xml",
 754                                         self.username, self.password)
 755                 sout, serr = locfg.run_noexcept(cmd)
 756
 757                 if sout.strip() != "" or serr.strip() != "":
 758                         print "sout: %s" % sout.strip()
 759                         return sout.strip() + serr.strip()
 760
 761                 if not dryrun:
 762                         locfg = command.CMD()
 763                         cmd = cmd_str + "locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 764                                                 self.host, cmd_str+"iloxml/Reset_Server.xml",
 765                                                 self.username, self.password)
 766                         sout, serr = locfg.run_noexcept(cmd)
 767
 768                         if sout.strip() != "":
 769                                 print "sout: %s" % sout.strip()
 770                                 #return sout.strip()
 771
 772                 return 0
 773
 774 class BayTechRPC3NC(PCUControl):
 775         def run(self, node_port, dryrun):
 776                 self.open(self.host, self.username, None, "Enter user name:")
 777                 self.sendPassword(self.password, "Enter Password:")
 778
 779                 #self.ifThenSend("RPC-16>", "Status")
 780                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 781
 782                 # Reboot Outlet  N        (Y/N)?
 783                 if dryrun:
 784                         self.ifThenSend("(Y/N)?", "N")
 785                 else:
 786                         self.ifThenSend("(Y/N)?", "Y")
 787                 self.ifThenSend("RPC3-NC>", "")
 788
 789                 self.close()
 790                 return 0
 791
 792 class BayTechRPC16(PCUControl):
 793         def run(self, node_port, dryrun):
 794                 self.open(self.host, self.username, None, "Enter user name:")
 795                 self.sendPassword(self.password, "Enter Password:")
 796
 797                 #self.ifThenSend("RPC-16>", "Status")
 798
 799                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 800
 801                 # Reboot Outlet  N        (Y/N)?
 802                 if dryrun:
 803                         self.ifThenSend("(Y/N)?", "N")
 804                 else:
 805                         self.ifThenSend("(Y/N)?", "Y")
 806                 self.ifThenSend("RPC-16>", "")
 807
 808                 self.close()
 809                 return 0
 810
 811 class BayTechCtrlCUnibe(PCUControl):
 812         """
 813                 For some reason, these units let you log in fine, but they hang
 814                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 815                 why.
 816         """
 817         def run(self, node_port, dryrun):
 818                 print "BayTechCtrlC %s" % self.host
 819
 820                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 821                 s = pxssh.pxssh()
 822                 if not s.login(self.host, self.username, self.password, ssh_options):
 823                         raise ExceptionPassword("Invalid Password")
 824                 # Otherwise, the login succeeded.
 825
 826                 # Send a ctrl-c to the remote process.
 827                 print "sending ctrl-c"
 828                 s.send(chr(3))
 829
 830                 # Control Outlets  (5 ,1).........5
 831                 try:
 832                         #index = s.expect("Enter Request")
 833                         index = s.expect(["Enter Request :"])
 834
 835                         if index == 0:
 836                                 print "3"
 837                                 s.send("3\r\n")
 838                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 839                                 if index == 1:
 840                                         s.send(self.username + "\r\n")
 841                                         index = s.expect(["DS-RPC>"])
 842
 843                                 if index == 0:
 844                                         print "Reboot %d" % node_port
 845                                         time.sleep(5)
 846                                         s.send("Reboot %d\r\n" % node_port)
 847
 848                                         time.sleep(5)
 849                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 850                                         if index == 0:
 851                                                 if dryrun:
 852                                                         print "sending N"
 853                                                         s.send("N\r\n")
 854                                                 else:
 855                                                         print "sending Y"
 856                                                         s.send("Y\r\n")
 857                                         elif index == 1:
 858                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 859                                         elif index == 2:
 860                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 861
 862                                 time.sleep(5)
 863                                 index = s.expect(["DS-RPC>"])
 864                                 #print "got prompt back"
 865
 866                         s.close()
 867
 868                 except pexpect.EOF:
 869                         raise ExceptionPrompt("EOF before expected Prompt")
 870                 except pexpect.TIMEOUT:
 871                         raise ExceptionPrompt("Timeout before expected Prompt")
 872
 873                 return 0
 874
 875 class BayTechCtrlC(PCUControl):
 876         """
 877                 For some reason, these units let you log in fine, but they hang
 878                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 879                 why.
 880         """
 881         def run(self, node_port, dryrun):
 882                 print "BayTechCtrlC %s" % self.host
 883
 884                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 885                 s = pxssh.pxssh()
 886                 if not s.login(self.host, self.username, self.password, ssh_options):
 887                         raise ExceptionPassword("Invalid Password")
 888                 # Otherwise, the login succeeded.
 889
 890                 # Send a ctrl-c to the remote process.
 891                 print "SENDING ctrl-c"
 892                 s.send(chr(3))
 893
 894                 # Control Outlets  (5 ,1).........5
 895                 try:
 896                         print "EXPECTING: ", "Enter Request :"
 897                         index = s.expect(["Enter Request :"])
 898
 899                         if index == 0:
 900                                 print "SENDING: 5"
 901                                 s.send("5\r\n")
 902                                 print "EXPECTING: ", "DS-RPC>"
 903                                 index = s.expect(["DS-RPC>", "Enter user name:", "Port in use."])
 904                                 if index == 1:
 905                                         print "sending username"
 906                                         s.send(self.username + "\r\n")
 907                                         index = s.expect(["DS-RPC>"])
 908                                 elif index == 2:
 909                                         raise ExceptionPrompt("PCU Reported 'Port in use.'")
 910
 911                                 if index == 0:
 912                                         print "SENDING: Reboot %d" % node_port
 913                                         s.send("Reboot %d\r\n" % node_port)
 914
 915                                         print "SLEEPING: 5"
 916                                         time.sleep(5)
 917                                         print "EXPECTING: ", "Y/N?"
 918                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 919                                         if index == 0:
 920                                                 if dryrun:
 921                                                         print "sending N"
 922                                                         s.send("N\r\n")
 923                                                 else:
 924                                                         print "SENDING: Y"
 925                                                         s.send("Y\r\n")
 926                                         elif index == 1:
 927                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 928                                         elif index == 2:
 929                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 930
 931                                 # NOTE: for some reason, the script times out with the
 932                                 # following line.  In manual tests, it works correctly, but
 933                                 # with automated tests, evidently it fails.
 934                                 print "SLEEPING: 5"
 935                                 time.sleep(5)
 936                                 #print "TOTAL--", s.allstr, "--EOT"
 937                                 index = s.expect(["DS-RPC>"])
 938                                 print "got prompt back"
 939
 940                         s.close()
 941
 942                 except pexpect.EOF:
 943                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 944                 except pexpect.TIMEOUT:
 945                         raise ExceptionPrompt("Timeout before Prompt")
 946
 947                 return 0
 948
 949 class BayTech(PCUControl):
 950         supported_ports = [22,23]
 951         def run(self, node_port, dryrun):
 952                 self.open(self.host, self.username)
 953                 self.sendPassword(self.password)
 954
 955                 # Control Outlets  (5 ,1).........5
 956                 self.ifThenSend("Enter Request :", "5")
 957
 958                 # Reboot N
 959                 try:
 960                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 961                 except ExceptionNotFound, msg:
 962                         # one machine is configured to ask for a username,
 963                         # even after login...
 964                         print "msg: %s" % msg
 965                         self.transport.write(self.username + "\r\n")
 966                         time.sleep(5)
 967                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 968
 969                 # Reboot Outlet  N        (Y/N)?
 970                 if dryrun:
 971                         self.ifThenSend("(Y/N)?", "N")
 972                 else:
 973                         self.ifThenSend("(Y/N)?", "Y")
 974                 time.sleep(5)
 975                 self.ifThenSend("DS-RPC>", "")
 976
 977                 self.close()
 978                 return 0
 979
 980 class WTIIPS4(PCUControl):
 981         supported_ports = [23]
 982         def run(self, node_port, dryrun):
 983                 self.open(self.host)
 984                 self.sendPassword(self.password, "Enter Password:")
 985
 986                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 987                 if not dryrun:
 988                         self.ifThenSend("Sure? (Y/N): ", "N")
 989                 else:
 990                         self.ifThenSend("Sure? (Y/N): ", "Y")
 991
 992                 self.ifThenSend("IPS> ", "")
 993
 994                 self.close()
 995                 return 0
 996
 997 class ePowerSwitchNew(PCUControl):
 998         # NOTE:
 999         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
1000         #               For some reason this both doesn't work and in some cases, actually
1001         #               hangs the PCU.  Definitely not what we want.
1002         #
1003         #               The code below is much simpler.  Just letting things fail first,
1004         #               and then, trying again with authentication string in the header.
1005         #
1006         def run(self, node_port, dryrun):
1007                 self.transport = None
1008                 self.url = "http://%s:%d/" % (self.host,80)
1009                 uri = "%s:%d" % (self.host,80)
1010
1011                 req = urllib2.Request(self.url)
1012                 try:
1013                         handle = urllib2.urlopen(req)
1014                 except IOError, e:
1015                         # NOTE: this is expected to fail initially
1016                         pass
1017                 else:
1018                         print self.url
1019                         print "-----------"
1020                         print handle.read()
1021                         print "-----------"
1022                         return "ERROR: not protected by HTTP authentication"
1023
1024                 if not hasattr(e, 'code') or e.code != 401:
1025                         return "ERROR: failed for: %s" % str(e)
1026
1027                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
1028                 # NOTE: assuming basic realm authentication.
1029                 authheader = "Basic %s" % base64data
1030                 req.add_header("Authorization", authheader)
1031
1032                 try:
1033                         f = urllib2.urlopen(req)
1034                 except IOError, e:
1035                         # failing here means the User/passwd is wrong (hopefully)
1036                         raise ExceptionPassword("Incorrect username/password")
1037
1038                 # NOTE: after verifying that the user/password is correct,
1039                 #               actually reboot the given node.
1040                 if not dryrun:
1041                         try:
1042                                 data = urllib.urlencode({'P%d' % node_port : "r"})
1043                                 req = urllib2.Request(self.url + "cmd.html")
1044                                 req.add_header("Authorization", authheader)
1045                                 # add data to handler,
1046                                 f = urllib2.urlopen(req, data)
1047                                 if self.verbose: print f.read()
1048                         except:
1049                                 import traceback; traceback.print_exc()
1050
1051                                 # fetch url one more time on cmd.html, econtrol.html or whatever.
1052                                 # pass
1053                 else:
1054                         if self.verbose: print f.read()
1055
1056                 self.close()
1057                 return 0
1058
1059 class CustomPCU(PCUControl):
1060         def run(self, node_port, dryrun):
1061                 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
1062
1063                 if not dryrun:
1064                         # Turn host off, then on
1065                         formstr = "plab%s=off" % node_port
1066                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
1067                         time.sleep(5)
1068                         formstr = "plab%s=on" % node_port
1069                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
1070                 else:
1071                         os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
1072
1073
1074 class ePowerSwitchOld(PCUControl):
1075         def run(self, node_port, dryrun):
1076                 self.url = "http://%s:%d/" % (self.host,80)
1077                 uri = "%s:%d" % (self.host,80)
1078
1079                 # create authinfo
1080                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1081                 authinfo.add_password (None, uri, self.username, self.password)
1082                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1083
1084                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
1085                 transport = urllib2.build_opener(authinfo)
1086                 f = transport.open(self.url)
1087                 if self.verbose: print f.read()
1088
1089                 if not dryrun:
1090                         transport = urllib2.build_opener(authhandler)
1091                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
1092                         if self.verbose: print f.read()
1093
1094                 self.close()
1095                 return 0
1096
1097 class ePowerSwitchOld(PCUControl):
1098         supported_ports = [80]
1099         def run(self, node_port, dryrun):
1100                 self.url = "http://%s:%d/" % (self.host,80)
1101                 uri = "%s:%d" % (self.host,80)
1102
1103                 # TODO: I'm still not sure what the deal is here.
1104                 #               two independent calls appear to need to be made before the
1105                 #               reboot will succeed.  It doesn't seem to be possible to do
1106                 #               this with a single call.  I have no idea why.
1107
1108                 # create authinfo
1109                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1110                 authinfo.add_password (None, uri, self.username, self.password)
1111                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1112
1113                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
1114                 transport = urllib2.build_opener()
1115                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
1116                 if self.verbose: print f.read()
1117
1118                 if not dryrun:
1119                         transport = urllib2.build_opener(authhandler)
1120                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
1121                         if self.verbose: print f.read()
1122
1123                 #       data= "P%d=r" % node_port
1124                 #self.open(self.host, self.username, self.password)
1125                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
1126                 #self.sendHTTP("econtrol.html", data)
1127                 #self.sendHTTP("cmd.html", data)
1128
1129                 self.close()
1130                 return 0
1131
1132 class ManualPCU(PCUControl):
1133         supported_ports = [22,23,80,443]
1134
1135         def run(self, node_port, dryrun):
1136                 if not dryrun:
1137                         # TODO: send email message to monitor admin requesting manual
1138                         # intervention.  This should always be an option for ridiculous,
1139                         # custom jobs.
1140                         pass
1141                 return 0
1142
1143 class PM211MIP(ManualPCU):
1144         supported_ports = [80,443]
1145
1146 ### rebooting european BlackBox PSE boxes
1147 # Thierry Parmentelat - May 11 2005
1148 # tested on 4-ports models known as PSE505-FR
1149 # uses http to POST a data 'P<port>=r'
1150 # relies on basic authentication within http1.0
1151 # first curl-based script was
1152 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
1153 #       http://<hostname>:<http_port>/cmd.html && echo OK
1154
1155 # log in:
1156
1157 ## BB PSMaverick
1158 class BlackBoxPSMaverick(PCUControl):
1159         supported_ports = [80]
1160
1161         def run(self, node_port, dryrun):
1162                 if not dryrun:
1163                         # send reboot signal.
1164                         cmd = "curl -s --data 'P%s=r' --anyauth --user '%s:%s' http://%s/config/home_f.html" % ( node_port, self.username, self.password, self.host)
1165                 else:
1166                         # else, just try to log in
1167                         cmd = "curl -s --anyauth --user '%s:%s' http://%s/config/home_f.html" % ( self.username, self.password, self.host)
1168
1169                 p = os.popen(cmd)
1170                 result = p.read()
1171                 print "RESULT: ", result
1172
1173                 if len(result.split()) > 3:
1174                         return 0
1175                 else:
1176                         return result
1177
1178 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
1179
1180         global verbose
1181
1182         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
1183         data= "P%d=r" % port_in_pcu
1184         if verbose:
1185                 logger.debug("POSTing '%s' on %s" % (data,url))
1186
1187         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1188         uri = "%s:%d" % (pcu_ip,http_port)
1189         authinfo.add_password (None, uri, username, password)
1190         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1191
1192         opener = urllib2.build_opener(authhandler)
1193         urllib2.install_opener(opener)
1194
1195         if (dryrun):
1196                 return 0
1197
1198         try:
1199                 f = urllib2.urlopen(url,data)
1200
1201                 r= f.read()
1202                 if verbose:
1203                         logger.debug(r)
1204                 return 0
1205
1206         except urllib2.URLError,err:
1207                 logger.info('Could not open http connection', err)
1208                 return "bbpse error"
1209
1210 ### rebooting x10toggle based systems addressed by port
1211 # Marc E. Fiuczynski - May 31 2005
1212 # tested on 4-ports models known as PSE505-FR
1213 # uses ssh and password to login to an account
1214 # that will cause the system to be powercycled.
1215
1216 def x10toggle_reboot(ip, username, password, port, dryrun):
1217         global verbose
1218
1219         ssh = None
1220         try:
1221                 ssh = pyssh.Ssh(username, ip)
1222                 ssh.open()
1223
1224                 # Login
1225                 telnet_answer(ssh, "password:", password)
1226
1227                 if not dryrun:
1228                         # Reboot
1229                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
1230
1231                 # Close
1232                 output = ssh.close()
1233                 if verbose:
1234                         logger.debug(output)
1235                 return 0
1236
1237         except Exception, err:
1238                 if verbose:
1239                         logger.debug(err)
1240                 if ssh:
1241                         output = ssh.close()
1242                         if verbose:
1243                                 logger.debug(output)
1244                 return errno.ETIMEDOUT
1245
1246 ### rebooting Dell systems via RAC card
1247 # Marc E. Fiuczynski - June 01 2005
1248 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1249 #
1250
1251 def runcmd(command, args, username, password, timeout = None):
1252
1253         result = [None]
1254         result_ready = threading.Condition()
1255
1256         def set_result(x):
1257
1258                 result_ready.acquire()
1259                 try:
1260                         result[0] = x
1261                 finally:
1262                         result_ready.notify()
1263                         result_ready.release()
1264
1265         def do_command(command, username, password):
1266
1267                 try:
1268                         # Popen4 is a popen-type class that combines stdout and stderr
1269                         p = popen2.Popen4(command)
1270
1271                         # read all output data
1272                         p.tochild.write("%s\n" % username)
1273                         p.tochild.write("%s\n" % password)
1274                         p.tochild.close()
1275                         data = p.fromchild.read()
1276
1277                         while True:
1278                                 # might get interrupted by a signal in poll() or waitpid()
1279                                 try:
1280                                         retval = p.wait()
1281                                         set_result((retval, data))
1282                                         break
1283                                 except OSError, ex:
1284                                         if ex.errno == errno.EINTR:
1285                                                 continue
1286                                         raise ex
1287                 except Exception, ex:
1288                         set_result(ex)
1289
1290         if args:
1291                 command = " ".join([command] + args)
1292
1293         worker = threading.Thread(target = do_command, args = (command, username, password, ))
1294         worker.setDaemon(True)
1295         result_ready.acquire()
1296         worker.start()
1297         result_ready.wait(timeout)
1298         try:
1299                 if result == [None]:
1300                         raise Exception, "command timed-out: '%s'" % command
1301         finally:
1302                 result_ready.release()
1303         result = result[0]
1304
1305         if isinstance(result, Exception):
1306                 raise result
1307         else:
1308                 (retval, data) = result
1309                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1310                         return data
1311                 else:
1312                         out = "system command ('%s') " % command
1313                         if os.WIFEXITED(retval):
1314                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1315                         else:
1316                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1317                         if data:
1318                                 out += "; output follows:\n" + data
1319                         raise Exception, out
1320
1321 def racadm_reboot(host, username, password, port, dryrun):
1322         global verbose
1323
1324         ip = socket.gethostbyname(host)
1325         try:
1326                 cmd = "/usr/sbin/racadm"
1327                 os.stat(cmd)
1328                 if not dryrun:
1329                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1330                                 username, password)
1331                 else:
1332                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1333                                 username, password)
1334
1335                 print "RUNCMD: %s" % output
1336                 if verbose:
1337                         logger.debug(output)
1338                 return 0
1339
1340         except Exception, err:
1341                 logger.debug("runcmd raised exception %s" % err)
1342                 if verbose:
1343                         logger.debug(err)
1344                 return err
1345
1346 def pcu_name(pcu):
1347         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1348                 return pcu['hostname']
1349         elif pcu['ip'] is not None and pcu['ip'] is not "":
1350                 return pcu['ip']
1351         else:
1352                 return None
1353
1354
1355 def get_pcu_values(pcu_id):
1356         print "pcuid: %s" % pcu_id
1357         try:
1358                 pcurec = FindbadPCURecord.get_latest_by(plc_pcuid=pcu_id).first()
1359                 if pcurec:
1360                         values = pcurec.to_dict()
1361                 else:
1362                         values = None
1363         except:
1364                 values = None
1365
1366         return values
1367
1368 def reboot(nodename):
1369         return reboot_policy(nodename, True, False)
1370
1371 def reboot_str(nodename):
1372         global verbose
1373         continue_probe = True
1374         dryrun=False
1375
1376         pcu = plc.getpcu(nodename)
1377         if not pcu:
1378                 logger.debug("no pcu for %s" % nodename)
1379                 print "no pcu for %s" % nodename
1380                 return False # "%s has no pcu" % nodename
1381
1382         values = get_pcu_values(pcu['pcu_id'])
1383         if values == None:
1384                 logger.debug("No values for pcu probe %s" % nodename)
1385                 print "No values for pcu probe %s" % nodename
1386                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1387
1388         # Try the PCU first
1389         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1390
1391         ret = reboot_test_new(nodename, values, verbose, dryrun)
1392         return ret
1393
1394 def reboot_policy(nodename, continue_probe, dryrun):
1395         global verbose
1396
1397         pcu = plc.getpcu(nodename)
1398         if not pcu:
1399                 logger.debug("no pcu for %s" % nodename)
1400                 print "no pcu for %s" % nodename
1401                 return False # "%s has no pcu" % nodename
1402
1403         values = get_pcu_values(pcu['pcu_id'])
1404         if values == None:
1405                 logger.debug("No values for pcu probe %s" % nodename)
1406                 print "No values for pcu probe %s" % nodename
1407                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1408
1409         # Try the PCU first
1410         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1411
1412         ret = reboot_test_new(nodename, values, verbose, dryrun)
1413
1414         if ret != 0:
1415                 print ret
1416                 return False
1417         else:
1418                 print "return true"
1419                 return True
1420
1421 class Unknown(PCUControl):
1422         supported_ports = [22,23,80,443,5869,9100,16992]
1423
1424 def model_to_object(modelname):
1425         if modelname is None:
1426                 return ManualPCU
1427         if "AMT" in modelname:
1428                 return IntelAMT
1429         elif "BayTech" in modelname:
1430                 return BayTech
1431         elif "HPiLO" in modelname:
1432                 return HPiLO
1433         elif "IPAL" in modelname:
1434                 return IPAL
1435         elif "APC" in modelname:
1436                 return APCControl
1437         elif "DRAC" in modelname:
1438                 return DRAC
1439         elif "WTI" in modelname:
1440                 return WTIIPS4
1441         elif "ePowerSwitch" in modelname:
1442                 return ePowerSwitchNew
1443         elif "IPMI" in modelname:
1444                 return IPMI
1445         elif "BlackBoxPSMaverick" in modelname:
1446                 return BlackBoxPSMaverick
1447         elif "PM211MIP" in modelname:
1448                 return PM211MIP
1449         elif "ManualPCU" in modelname:
1450                 return ManualPCU
1451         else:
1452                 print "UNKNOWN model %s"%modelname
1453                 return Unknown
1454
1455 def reboot_test_new(nodename, values, verbose, dryrun):
1456         rb_ret = ""
1457         if 'plc_pcu_stats' in values:
1458                 values.update(values['plc_pcu_stats'])
1459
1460         try:
1461                 modelname = values['model']
1462                 if modelname:
1463                         object = eval('%s(values, verbose, ["22", "23", "80", "443", "9100", "16992", "5869"])' % modelname)
1464                         rb_ret = object.reboot(values[nodename], dryrun)
1465                 else:
1466                         rb_ret =  "Not_Run"
1467                 # TODO: how to handle the weird, georgetown pcus, the drac faults, and ilo faults
1468         except ExceptionPort, err:
1469                 rb_ret = str(err)
1470
1471         return rb_ret
1472
1473
1474 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1475         rb_ret = ""
1476         if 'plc_pcu_stats' in values:
1477                 values.update(values['plc_pcu_stats'])
1478
1479         try:
1480                 # DataProbe iPal (many sites)
1481                 if  continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1482                         ipal = IPAL(values, verbose, ['23', '80', '9100'])
1483                         rb_ret = ipal.reboot(values[nodename], dryrun)
1484
1485                 # APC Masterswitch (Berkeley)
1486                 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1487                                                                   values['model'].find("Masterswitch") >= 0 ):
1488                         print values
1489
1490                         # TODO: make a more robust version of APC
1491                         if values['pcu_id'] in [1102,1163,1055,1111,1231,1113,1127,1128,1148]:
1492                                 apc = APCControl12p3(values, verbose, ['22', '23'])
1493                                 rb_ret = apc.reboot(values[nodename], dryrun)
1494
1495                         elif values['pcu_id'] in [1110,86]:
1496                                 apc = APCControl1p4(values, verbose, ['22', '23'])
1497                                 rb_ret = apc.reboot(values[nodename], dryrun)
1498
1499                         elif values['pcu_id'] in [1221,1225,1220,1192]:
1500                                 apc = APCControl121p3(values, verbose, ['22', '23'])
1501                                 rb_ret = apc.reboot(values[nodename], dryrun)
1502
1503                         elif values['pcu_id'] in [1173,1240,47,1363,1405,1401,1372,1371]:
1504                                 apc = APCControl121p1(values, verbose, ['22', '23'])
1505                                 rb_ret = apc.reboot(values[nodename], dryrun)
1506
1507                         else:
1508                                 apc = APCControl13p13(values, verbose, ['22', '23'])
1509                                 rb_ret = apc.reboot(values[nodename], dryrun)
1510
1511                 # BayTech DS4-RPC
1512                 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1513                         if values['pcu_id'] in [1056,1237,1052,1209,1002,1008,1041,1013,1022]:
1514                                 # These  require a 'ctrl-c' to be sent...
1515                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1516                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1517
1518                         elif values['pcu_id'] in [93]:
1519                                 baytech = BayTechRPC3NC(values, verbose, ['22', '23'])
1520                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1521
1522                         elif values['pcu_id'] in [1057]:
1523                                 # These  require a 'ctrl-c' to be sent...
1524                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1525                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1526
1527                         elif values['pcu_id'] in [1012]:
1528                                 # This pcu sometimes doesn't present the 'Username' prompt,
1529                                 # unless you immediately try again...
1530                                 try:
1531                                         baytech = BayTechRPC16(values, verbose, ['22', '23'])
1532                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1533                                 except:
1534                                         baytech = BayTechRPC16(values, verbose, ['22', '23'])
1535                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1536                         else:
1537                                 baytech = BayTech(values, verbose, ['22', '23'])
1538                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1539
1540                 # iLO
1541                 elif continue_probe and values['model'].find("ilo") >= 0:
1542                         try:
1543                                 hpilo = HPiLO(values, verbose, ['22'])
1544                                 rb_ret = hpilo.reboot(0, dryrun)
1545                                 if rb_ret != 0:
1546                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1547                                         rb_ret = hpilo.reboot(0, dryrun)
1548                         except:
1549                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1550                                 rb_ret = hpilo.reboot(0, dryrun)
1551
1552                 # DRAC ssh
1553                 elif continue_probe and values['model'].find("DRAC") >= 0:
1554                         # TODO: I don't think DRAC will throw an exception for the
1555                         # default method to catch...
1556                         try:
1557                                 drac = DRAC(values, verbose, ['443', '5869'])
1558                                 rb_ret = drac.reboot(0, dryrun)
1559                         except:
1560                                 drac = DRACDefault(values, verbose, ['22'])
1561                                 rb_ret = drac.reboot(0, dryrun)
1562
1563                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1564                                 wti = WTIIPS4(values, verbose, ['23'])
1565                                 rb_ret = wti.reboot(values[nodename], dryrun)
1566
1567                 elif continue_probe and values['model'].find("AMT") >= 0:
1568                                 amt = IntelAMT(values, verbose, ['16992'])
1569                                 rb_ret = amt.reboot(values[nodename], dryrun)
1570
1571                 elif continue_probe and values['model'].find("bbsemaverick") >=0:
1572                         print "TRYING BlackBoxPSMaverick"
1573                         bbe = BlackBoxPSMaverick(values, verbose, ['80'])
1574                         rb_ret = bbe.reboot(values[nodename], dryrun)
1575
1576                 elif continue_probe and values['model'].find("ipmi") >=0:
1577
1578                         print "TRYING IPMI"
1579                         ipmi = IPMI(values, verbose, ['80', '443', '623'])
1580                         rb_ret = ipmi.reboot(values[nodename], dryrun)
1581
1582                 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1583                         # TODO: allow a different port than http 80.
1584                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1585                                 eps = ePowerSwitchNew(values, verbose, ['80'])
1586                         elif values['pcu_id'] in [1003]:
1587                                 # OLD EPOWER
1588                                 print "OLD EPOWER"
1589                                 eps = ePowerSwitchOld(values, verbose, ['80'])
1590                         else:
1591                                 eps = ePowerSwitchNew(values, verbose, ['80'])
1592
1593                         rb_ret = eps.reboot(values[nodename], dryrun)
1594                 elif continue_probe and values['pcu_id'] in [1122]:
1595                         custom = CustomPCU(values, verbose, ['80', '443'])
1596                         custom.reboot(values[nodename], dryrun)
1597
1598                 elif continue_probe:
1599                         rb_ret = "Unsupported_PCU"
1600
1601                 elif continue_probe == False:
1602                         if 'port_status' in values:
1603                                 rb_ret = "NetDown"
1604                         else:
1605                                 rb_ret = "Not_Run"
1606                 else:
1607                         rb_ret = -1
1608
1609         except ExceptionPort, err:
1610                 rb_ret = str(err)
1611
1612         return rb_ret
1613         # ????
1614         #elif continue_probe and values['protocol'] == "racadm" and \
1615         #               values['model'] == "RAC":
1616         #       rb_ret = racadm_reboot(pcu_name(values),
1617         #                                                                 values['username'],
1618         #                                                                 values['password'],
1619         #                                                                 pcu[nodename],
1620         #                                                                 dryrun)
1621
1622 def main():
1623         logger.setLevel(logging.DEBUG)
1624         ch = logging.StreamHandler()
1625         ch.setLevel(logging.DEBUG)
1626         formatter = logging.Formatter('LOGGER - %(message)s')
1627         ch.setFormatter(formatter)
1628         logger.addHandler(ch)
1629
1630         try:
1631                 if "test" in sys.argv:
1632                         dryrun = True
1633                 else:
1634                         dryrun = False
1635
1636                 for node in sys.argv[1:]:
1637                         if node == "test": continue
1638
1639                         print "Rebooting %s" % node
1640                         if reboot_policy(node, True, dryrun):
1641                                 print "success"
1642                         else:
1643                                 print "failed"
1644         except Exception, err:
1645                 import traceback; traceback.print_exc()
1646                 print err
1647
1648 if __name__ == '__main__':
1649         logger = logging.getLogger("monitor")
1650         main()