pcucontrol/reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import urllib
  12 import threading, popen2
  13 import array, struct
  14 from monitor.wrapper import plc
  15 import base64
  16 from subprocess import PIPE, Popen
  17 import pcucontrol.transports.ssh.pxssh as pxssh
  18 import pcucontrol.transports.ssh.pexpect as pexpect
  19 import socket
  20 from monitor.util import command
  21
  22 # Use our versions of telnetlib and pyssh
  23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  24 import pcucontrol.transports.telnetlib as telnetlib
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  26 import pcucontrol.transports.pyssh as pyssh
  27 from monitor import config
  28
  29 from monitor.database.info.model import FindbadPCURecord
  30
  31 # Timeouts in seconds
  32 TELNET_TIMEOUT = 45
  33
  34 # Event class ID from pcu events
  35 #NODE_POWER_CONTROL = 3
  36
  37 # Monitor user ID
  38 #MONITOR_USER_ID = 11142
  39
  40 import logging
  41 logger = logging.getLogger("monitor")
  42 verbose = 1
  43 #dryrun = 0;
  44
  45 class ExceptionNoTransport(Exception): pass
  46 class ExceptionNotFound(Exception): pass
  47 class ExceptionPassword(Exception): pass
  48 class ExceptionTimeout(Exception): pass
  49 class ExceptionPrompt(Exception): pass
  50 class ExceptionSequence(Exception): pass
  51 class ExceptionReset(Exception): pass
  52 class ExceptionPort(Exception): pass
  53 class ExceptionUsername(Exception): pass
  54
  55 def telnet_answer(telnet, expected, buffer):
  56         global verbose
  57
  58         output = telnet.read_until(expected, TELNET_TIMEOUT)
  59         #if verbose:
  60         #       logger.debug(output)
  61         if output.find(expected) == -1:
  62                 raise ExceptionNotFound, "'%s' not found" % expected
  63         else:
  64                 telnet.write(buffer + "\r\n")
  65
  66
  67 # PCU has model, host, preferred-port, user, passwd,
  68
  69 # This is an object derived directly form the PLCAPI DB fields
  70 class PCU(object):
  71         def __init__(self, plc_pcu_dict):
  72                 for field in ['username', 'password', 'site_id',
  73                                                 'hostname', 'ip',
  74                                                 'pcu_id', 'model',
  75                                                 'node_ids', 'ports', ]:
  76                         if field in plc_pcu_dict:
  77                                 self.__setattr__(field, plc_pcu_dict[field])
  78                         else:
  79                                 raise Exception("No such field %s in PCU object" % field)
  80
  81 # These are the convenience functions build around the PCU object.
  82 class PCUModel(PCU):
  83         def __init__(self, plc_pcu_dict):
  84                 PCU.__init__(self, plc_pcu_dict)
  85                 self.host = self.pcu_name()
  86
  87         def pcu_name(self):
  88                 if self.hostname is not None and self.hostname is not "":
  89                         return self.hostname
  90                 elif self.ip is not None and self.ip is not "":
  91                         return self.ip
  92                 else:
  93                         return None
  94
  95         def nodeidToPort(self, node_id):
  96                 if node_id in self.node_ids:
  97                         for i in range(0, len(self.node_ids)):
  98                                 if node_id == self.node_ids[i]:
  99                                         return self.ports[i]
 100
 101                 raise Exception("No such Node ID: %d" % node_id)
 102
 103 # This class captures the observed pcu records from FindBadPCUs.py
 104 class PCURecord:
 105         def __init__(self, pcu_record_dict):
 106                 for field in ['port_status',
 107                                                 'dns_status',
 108                                                 'entry_complete', ]:
 109                         if field in pcu_record_dict:
 110                                 if field == "reboot":
 111                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 112                                 else:
 113                                         self.__setattr__(field, pcu_record_dict[field])
 114                         else:
 115                                 raise Exception("No such field %s in pcu record dict" % field)
 116
 117 class Transport:
 118         TELNET = 1
 119         SSH    = 2
 120         HTTP   = 3
 121         HTTPS  = 4
 122         IPAL   = 5
 123
 124         TELNET_TIMEOUT = 120
 125
 126         def __init__(self, type, verbose):
 127                 self.type = type
 128                 self.verbose = verbose
 129                 self.transport = None
 130
 131         def open(self, host, username=None, password=None, prompt="User Name"):
 132                 transport = None
 133
 134                 if self.type == self.TELNET:
 135                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 136                         transport.set_debuglevel(self.verbose)
 137                         if username is not None:
 138                                 self.transport = transport
 139                                 self.ifThenSend(prompt, username, ExceptionUsername)
 140
 141                 elif self.type == self.SSH:
 142                         if username is not None:
 143                                 transport = pyssh.Ssh(username, host)
 144                                 transport.set_debuglevel(self.verbose)
 145                                 transport.open()
 146                                 # TODO: have an ssh set_debuglevel() also...
 147                         else:
 148                                 raise Exception("Username cannot be None for ssh transport.")
 149                 elif self.type == self.HTTP:
 150                         self.url = "http://%s:%d/" % (host,80)
 151                         uri = "%s:%d" % (host,80)
 152
 153                         # create authinfo
 154                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 155                         authinfo.add_password (None, uri, username, password)
 156                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 157
 158                         transport = urllib2.build_opener(authhandler)
 159
 160                 else:
 161                         raise Exception("Unknown transport type: %s" % self.type)
 162
 163                 self.transport = transport
 164                 return True
 165
 166         def close(self):
 167                 if self.type == self.TELNET:
 168                         self.transport.close()
 169                 elif self.type == self.SSH:
 170                         self.transport.close()
 171                 elif self.type == self.HTTP:
 172                         pass
 173                 else:
 174                         raise Exception("Unknown transport type %s" % self.type)
 175                 self.transport = None
 176
 177         def sendHTTP(self, resource, data):
 178                 if self.verbose:
 179                         print "POSTing '%s' to %s" % (data,self.url + resource)
 180
 181                 try:
 182                         f = self.transport.open(self.url + resource ,data)
 183                         r = f.read()
 184                         if self.verbose:
 185                                 print r
 186
 187                 except urllib2.URLError,err:
 188                         logger.info('Could not open http connection', err)
 189                         return "http transport error"
 190
 191                 return 0
 192
 193         def sendPassword(self, password, prompt=None):
 194                 if self.type == self.TELNET:
 195                         if prompt == None:
 196                                 self.ifThenSend("Password", password, ExceptionPassword)
 197                         else:
 198                                 self.ifThenSend(prompt, password, ExceptionPassword)
 199                 elif self.type == self.SSH:
 200                         self.ifThenSend("password:", password, ExceptionPassword)
 201                 elif self.type == self.HTTP:
 202                         pass
 203                 else:
 204                         raise Exception("Unknown transport type: %s" % self.type)
 205
 206         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 207
 208                 if self.transport != None:
 209                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 210                         if output.find(expected) == -1:
 211                                 print "OUTPUT: --%s--" % output
 212                                 raise ErrorClass, "'%s' not found" % expected
 213                         else:
 214                                 self.transport.write(buffer + "\r\n")
 215                 else:
 216                         raise ExceptionNoTransport("transport object is type None")
 217
 218         def ifElse(self, expected, ErrorClass):
 219                 try:
 220                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 221                 except:
 222                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 223
 224
 225 class PCUControl(Transport,PCUModel,PCURecord):
 226
 227         supported_ports = []
 228
 229         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 230                 PCUModel.__init__(self, plc_pcu_record)
 231                 PCURecord.__init__(self, plc_pcu_record)
 232                 type = None
 233                 if self.port_status:
 234                         if '22' in supported_ports and self.port_status['22'] == "open":
 235                                 type = Transport.SSH
 236                         elif '23' in supported_ports and self.port_status['23'] == "open":
 237                                 type = Transport.TELNET
 238                         # NOTE: prefer https over http
 239                         elif '443' in supported_ports and self.port_status['443'] == "open":
 240                                 type = Transport.HTTPS
 241                         elif '80' in supported_ports and self.port_status['80'] == "open":
 242                                 type = Transport.HTTP
 243                         elif '5869' in supported_ports and self.port_status['5869'] == "open":
 244                                 # For DRAC cards. Racadm opens this port.
 245                                 type = Transport.HTTP
 246                         elif '9100' in supported_ports and self.port_status['9100'] == "open":
 247                                 type = Transport.IPAL
 248                         elif '16992' in supported_ports and self.port_status['16992'] == "open":
 249                                 type = Transport.HTTP
 250                         else:
 251                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 252                 else:
 253                         raise ExceptionPort("No Portstatus: No transport because no open ports")
 254                 Transport.__init__(self, type, verbose)
 255
 256         def run(self, node_port, dryrun):
 257                 """ This function is to be defined by the specific PCU instance.  """
 258                 pass
 259
 260         def reboot(self, node_port, dryrun):
 261                 try:
 262                         return self.run(node_port, dryrun)
 263                 except ExceptionNotFound, err:
 264                         return "error: " + str(err)
 265                 except ExceptionPassword, err:
 266                         return "Password exception: " + str(err)
 267                 except ExceptionTimeout, err:
 268                         return "Timeout exception: " + str(err)
 269                 except ExceptionUsername, err:
 270                         return "No username prompt: " + str(err)
 271                 except ExceptionSequence, err:
 272                         return "Sequence error: " + str(err)
 273                 except ExceptionPrompt, err:
 274                         return "Prompt exception: " + str(err)
 275                 except ExceptionNoTransport, err:
 276                         return "No Transport: " + str(err)
 277                 except ExceptionPort, err:
 278                         return "No ports exception: " + str(err)
 279                 except socket.error, err:
 280                         return "socket error: timeout: " + str(err)
 281                 except urllib2.HTTPError, err:
 282                         return "HTTPError: " + str(err)
 283                 except urllib2.URLError, err:
 284                         return "URLError: " + str(err)
 285                 except EOFError, err:
 286                         if self.verbose:
 287                                 logger.debug("reboot: EOF")
 288                                 logger.debug(err)
 289                         self.transport.close()
 290                         import traceback
 291                         traceback.print_exc()
 292                         return "EOF connection reset" + str(err)
 293
 294 class IPMI(PCUControl):
 295
 296         supported_ports = [80,443,623]
 297
 298         # TODO: get exit codes to determine success or failure...
 299         def run(self, node_port, dryrun):
 300
 301                 if not dryrun:
 302                         cmd = "ipmitool -I lanplus -H %s -U %s -P '%s' power cycle"
 303                         p = os.popen(cmd % ( self.host, self.username, self.password) )
 304                         result = p.read()
 305                         print "RESULT: ", result
 306                 else:
 307                         cmd = "ipmitool -I lanplus -H %s -U %s -P '%s' user list"
 308                         p = os.popen(cmd % ( self.host, self.username, self.password) )
 309                         result = p.read()
 310                         print "RESULT: ", result
 311
 312                 if "Error" in result:
 313                         return result
 314                 else:
 315                         return 0
 316
 317 class IPAL(PCUControl):
 318         """
 319                 This now uses a proprietary format for communicating with the PCU.  I
 320                 prefer it to Telnet, and Web access, since it's much lighter weight
 321                 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
 322         """
 323         supported_ports = [23,80,9100]
 324
 325         def format_msg(self, data, cmd):
 326                 esc = chr(int('1b',16))
 327                 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
 328
 329         def recv_noblock(self, s, count):
 330                 import errno
 331
 332                 try:
 333                         # TODO: make sleep backoff, before stopping.
 334                         time.sleep(4)
 335                         ret = s.recv(count, socket.MSG_DONTWAIT)
 336                 except socket.error, e:
 337                         if e[0] == errno.EAGAIN:
 338                                 raise Exception(e[1])
 339                         else:
 340                                 # TODO: not other exceptions.
 341                                 raise Exception(e)
 342                 return ret
 343
 344         def run(self, node_port, dryrun):
 345                 if self.type == Transport.IPAL:
 346                         return self.run_ipal(node_port, dryrun)
 347                 elif self.type == Transport.TELNET:
 348                         return self.run_telnet(node_port, dryrun)
 349                 else:
 350                         raise ExceptionNoTransport("Unimplemented Transport for IPAL")
 351
 352         def run_telnet(self, node_port, dryrun):
 353                 # TELNET version of protocol...
 354                 self.open(self.host)
 355                 ## XXX Some iPals require you to hit Enter a few times first
 356                 self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 357                 # Login
 358                 self.ifThenSend("Password >", self.password, ExceptionPassword)
 359                 self.transport.write("\r\n\r\n")
 360                 if not dryrun: # P# - Pulse relay
 361                         print "node_port %s" % node_port
 362                         self.ifThenSend("Enter >",
 363                                                         "P7", # % node_port,
 364                                                         ExceptionNotFound)
 365                         print "send newlines"
 366                         self.transport.write("\r\n\r\n")
 367                         print "after new lines"
 368                 # Get the next prompt
 369                 print "wait for enter"
 370                 self.ifElse("Enter >", ExceptionTimeout)
 371                 print "closing "
 372                 self.close()
 373                 return 0
 374
 375         def run_ipal(self, node_port, dryrun):
 376                 import errno
 377
 378                 power_on = False
 379
 380                 print "open socket"
 381                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 382                 try:
 383                         print "connect"
 384                         s.connect((self.host, 9100))
 385                 except socket.error, e:
 386                         s.close()
 387                         if e[0] == errno.ECONNREFUSED:
 388                                 # cannot connect to remote host
 389                                 raise Exception(e[1])
 390                         else:
 391                                 # TODO: what other conditions are there?
 392                                 raise Exception(e)
 393
 394                 # get current status
 395                 print "Checking status"
 396                 s.send(self.format_msg("", 'O'))
 397                 ret = self.recv_noblock(s, 8)
 398                 print "Current status is '%s'" % ret
 399
 400                 if ret == '':
 401                         raise Exception("Status returned 'another session already open' %s : %s" % (node_port, ret))
 402
 403                 if node_port < len(ret):
 404                         status = ret[node_port]
 405                         if status == '1':
 406                                 # up
 407                                 power_on = True
 408                         elif status == '0':
 409                                 # down
 410                                 power_on = False
 411                         else:
 412                                 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 413                 else:
 414                         raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 415
 416
 417                 if not dryrun:
 418                         if power_on:
 419                                 print "Pulsing %s" % node_port
 420                                 s.send(self.format_msg("%s" % node_port, 'P'))
 421                         else:
 422                                 # NOTE: turn power on ; do not pulse the port.
 423                                 print "Power was off, so turning on ..."
 424                                 s.send(self.format_msg("%s" % node_port, 'E'))
 425                                 #s.send(self.format_msg("%s" % node_port, 'P'))
 426
 427                         print "Receiving response."
 428                         ret = self.recv_noblock(s, 8)
 429                         print "Current status is '%s'" % ret
 430
 431                         if node_port < len(ret):
 432                                 status = ret[node_port]
 433                                 if status == '1':
 434                                         # up
 435                                         power_on = True
 436                                 elif status == '0':
 437                                         # down
 438                                         power_on = False
 439                                 else:
 440                                         raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
 441                         else:
 442                                 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
 443
 444                         if power_on:
 445                                 return 0
 446                         else:
 447                                 return "Failed Power On"
 448
 449                 s.close()
 450                 return 0
 451
 452 class APCControl(PCUControl):
 453         supported_ports = [22,23,80,443]
 454         reboot_sequence = []
 455
 456         def run(self, node_port, dryrun):
 457                 print "RUNNING!!!!!!!!!!!!"
 458                 if self.type == Transport.HTTPS or self.type == Transport.HTTP:
 459                         print "APC via http...."
 460                         return self.run_http_or_https(node_port, dryrun)
 461                 else:
 462                         print "APC via telnet/ssh...."
 463                         return self.run_telnet_or_ssh(node_port, dryrun)
 464
 465         def run_telnet_or_ssh(self, node_port, dryrun):
 466                 self.open(self.host, self.username)
 467                 self.sendPassword(self.password)
 468
 469                 first = True
 470                 for val in self.reboot_sequence:
 471                         if first:
 472                                 self.ifThenSend("\r\n> ", val, ExceptionPassword)
 473                                 first = False
 474                         else:
 475                                 self.ifThenSend("\r\n> ", val)
 476
 477                 if not dryrun:
 478                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 479                                                         "YES\r\n",
 480                                                         ExceptionSequence)
 481                 else:
 482                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 483                                                         "", ExceptionSequence)
 484                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 485
 486                 self.close()
 487                 return 0
 488
 489         def run_http_or_https(self, node_port, dryrun):
 490                 if not dryrun:
 491                         # send reboot signal.
 492                         # TODO: send a ManualPCU() reboot request for this PCU.
 493                         # NOTE: this model defies automation because, the port numbering
 494                         #       and the form numbers are not consistent across models.  There is
 495                         #       not direct mapping from port# to form#.
 496                         return "Manual Reboot Required"
 497
 498                 else:
 499                         # TODO: also send message for https, since that doesn't work this way...
 500                         if self.type == Transport.HTTPS:
 501                                 cmd = self.get_https_cmd()
 502                         elif self.type == Transport.HTTP:
 503                                 cmd = self.get_http_cmd()
 504                         else:
 505                                 raise ExceptionNoTransport("Unsupported transport for http command")
 506
 507                 cmd = cmd % ( self.username, self.password, self.host)
 508                 print "CMD: %s" % cmd
 509
 510                 p = os.popen(cmd)
 511                 result = p.read()
 512                 if len(result.split('\n')) > 2:
 513                         self.logout()
 514                         return 0
 515                 else:
 516                         # NOTE: an error has occurred, so no need to log out.
 517                         print "RESULT: ", result
 518                         return result
 519
 520         def get_https_cmd(self):
 521                 version = self.get_version()
 522                 print "VERSION: %s" % version
 523                 if "AP96" in version:
 524                         cmd = "curl -s --insecure --user '%s:%s' https://%s/outlets.htm " + \
 525                                   " | grep -E '^[^<]+' " + \
 526                                   " | grep -v 'Protected Object' "
 527                 else:
 528                         # NOTE: no other case known right now...
 529                         cmd = "curl -s --insecure --user '%s:%s' https://%s/outlets.htm " + \
 530                                   " | grep -E '^[^<]+' " + \
 531                                   " | grep -v 'Protected Object' "
 532
 533                 return cmd
 534
 535         def get_http_cmd(self):
 536                 version = self.get_version()
 537                 print "VERSION: %s" % version
 538                 if "AP7900" in version:
 539                         cmd = "curl -s --anyauth --user '%s:%s' http://%s/rPDUout.htm | grep -E '^[^<]+'"
 540                 elif "AP7920" in version:
 541                         cmd = "curl -s --anyauth --user '%s:%s' http://%s/ms3out.htm | grep -E '^[^<]+' "
 542                 else:
 543                         # default case...
 544                         print "USING DEFAULT"
 545                         cmd = "curl -s --anyauth --user '%s:%s' http://%s/ms3out.htm | grep -E '^[^<]+' "
 546
 547                 return cmd
 548
 549         def get_version(self):
 550                 # NOTE: this command returns and formats all data.
 551                 #cmd = """curl -s --anyauth --user '%s:%s' http://%s/about.htm """ +
 552                 #      """ | sed -e "s/<[^>]*>//g" -e "s/&nbsp;//g" -e "/^$/d" """ +
 553                 #         """ | awk '{line=$0 ; if ( ! /:/ && length(pline) > 0 ) \
 554                 #                            { print pline, line } else { pline=line} }' """ +
 555                 #         """ | grep Model """
 556
 557                 # NOTE: we may need to return software version, no model version to
 558                 #               know which file to request on the server.
 559
 560                 if self.type == Transport.HTTP:
 561                         cmd = """curl -s --anyauth --user '%s:%s' http://%s/about.htm """ + \
 562                                   """ | sed -e "s/<[^>]*>//g" -e "s/&nbsp;//g" -e "/^$/d" """ + \
 563                                   """ | grep -E "AP[[:digit:]]+" """
 564                                   #""" | grep -E "v[[:digit:]].*" """
 565                 elif self.type == Transport.HTTPS:
 566                         cmd = """curl -s --insecure --user '%s:%s' https://%s/about.htm """ + \
 567                                   """ | sed -e "s/<[^>]*>//g" -e "s/&nbsp;//g" -e "/^$/d" """ + \
 568                                   """ | grep -E "AP[[:digit:]]+" """
 569                                   #""" | grep -E "v[[:digit:]].*" """
 570                 else:
 571                         raise ExceptionNoTransport("Unsupported transport to get version")
 572
 573                 cmd = cmd % ( self.username, self.password, self.host)
 574                 p = os.popen(cmd)
 575                 result = p.read()
 576                 return result.strip()
 577
 578         def logout(self):
 579                 # NOTE: log out again, to allow other uses to access the machine.
 580                 if self.type == Transport.HTTP:
 581                         cmd = """curl -s --anyauth --user '%s:%s' http://%s/logout.htm """ + \
 582                                   """ | grep -E '^[^<]+' """
 583                 elif self.type == Transport.HTTPS:
 584                         cmd = """curl -s --insecure --user '%s:%s' http://%s/logout.htm """ + \
 585                                   """ | grep -E '^[^<]+' """
 586                 else:
 587                         raise ExceptionNoTransport("Unsupported transport to logout")
 588
 589                 cmd = cmd % ( self.username, self.password, self.host)
 590                 p = os.popen(cmd)
 591                 print p.read()
 592
 593 class APCControl12p3(APCControl):
 594         def run(self, node_port, dryrun):
 595                 self.reboot_sequence = ["1", "2", str(node_port), "3"]
 596                 return super(APCControl12p3, self).run(node_port, dryrun)
 597
 598 class APCControl1p4(APCControl):
 599         def run(self, node_port, dryrun):
 600                 self.reboot_sequence = ["1", str(node_port), "4"]
 601                 return super(APCControl1p4, self).run(node_port, dryrun)
 602
 603 class APCControl121p3(APCControl):
 604         def run(self, node_port, dryrun):
 605                 self.reboot_sequence = ["1", "2", "1", str(node_port), "3"]
 606                 return super(APCControl121p3, self).run(node_port, dryrun)
 607
 608 class APCControl121p1(APCControl):
 609         def run(self, node_port, dryrun):
 610                 self.reboot_sequence = ["1", "2", "1", str(node_port), "1", "3"]
 611                 return super(APCControl121p1, self).run(node_port, dryrun)
 612
 613 class APCControl13p13(APCControl):
 614         def run(self, node_port, dryrun):
 615                 self.reboot_sequence = ["1", "3", str(node_port), "1", "3"]
 616                 return super(APCControl13p13, self).run(node_port, dryrun)
 617
 618
 619 class IntelAMT(PCUControl):
 620         supported_ports = [16992]
 621
 622         def run(self, node_port, dryrun):
 623
 624                 cmd = command.CMD()
 625                 # TODO: need to make this path universal; not relative to pwd.
 626                 cmd_str = config.MONITOR_SCRIPT_ROOT + "/pcucontrol/models/intelamt/remoteControl"
 627
 628                 if dryrun:
 629                         # NOTE: -p checks the power state of the host.
 630                         # TODO: parse the output to find out if it's ok or not.
 631                         cmd_str += " -p http://%s:16992/RemoteControlService  -user admin -pass '%s' " % (self.host, self.password )
 632                 else:
 633                         cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
 634
 635                 print cmd_str
 636                 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
 637
 638 class DRAC(PCUControl):
 639         def run(self, node_port, dryrun):
 640
 641                 print "trying racadm_reboot..."
 642                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 643
 644                 return 0
 645
 646 class DRACDefault(PCUControl):
 647         supported_ports = [22,443,5869]
 648         def run(self, node_port, dryrun):
 649                 self.open(self.host, self.username)
 650                 self.sendPassword(self.password)
 651
 652                 print "logging in..."
 653                 self.transport.write("\r\n")
 654                 # Testing Reboot ?
 655                 if dryrun:
 656                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 657                 else:
 658                         # Reset this machine
 659                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 660
 661                 self.ifThenSend("[%s]#" % self.username, "exit")
 662
 663                 self.close()
 664                 return 0
 665
 666 class HPiLO(PCUControl):
 667         supported_ports = [22,443]
 668         def run(self, node_port, dryrun):
 669                 if self.type == Transport.SSH:
 670                         return self.run_ssh(node_port, dryrun)
 671                 elif self.type == Transport.HTTP or self.type == Transport.HTTPS:
 672                         return self.run_https(node_port, dryrun)
 673                 else:
 674                         raise ExceptionNoTransport("Unimplemented Transport for HPiLO %s" % self.type)
 675
 676         def run_ssh(self, node_port, dryrun):
 677
 678                 self.open(self.host, self.username)
 679                 self.sendPassword(self.password)
 680
 681                 # </>hpiLO->
 682                 self.ifThenSend("</>hpiLO->", "cd system1")
 683
 684                 # Reboot Outlet  N        (Y/N)?
 685                 if dryrun:
 686                         self.ifThenSend("</system1>hpiLO->", "POWER")
 687                 else:
 688                         # Reset this machine
 689                         self.ifThenSend("</system1>hpiLO->", "reset")
 690
 691                 self.ifThenSend("</system1>hpiLO->", "exit")
 692
 693                 self.close()
 694                 return 0
 695
 696         def run_https(self, node_port, dryrun):
 697
 698                 locfg = command.CMD()
 699
 700                 cmd_str = config.MONITOR_SCRIPT_ROOT + "/pcucontrol/models/hpilo/"
 701
 702                 cmd = cmd_str + "locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 703                                         self.host, cmd_str+"iloxml/Get_Network.xml",
 704                                         self.username, self.password)
 705                 sout, serr = locfg.run_noexcept(cmd)
 706
 707                 if sout.strip() != "" or serr.strip() != "":
 708                         print "sout: %s" % sout.strip()
 709                         return sout.strip() + serr.strip()
 710
 711                 if not dryrun:
 712                         locfg = command.CMD()
 713                         cmd = cmd_str + "locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 714                                                 self.host, cmd_str+"iloxml/Reset_Server.xml",
 715                                                 self.username, self.password)
 716                         sout, serr = locfg.run_noexcept(cmd)
 717
 718                         if sout.strip() != "":
 719                                 print "sout: %s" % sout.strip()
 720                                 #return sout.strip()
 721
 722                 return 0
 723
 724 class BayTechRPC3NC(PCUControl):
 725         def run(self, node_port, dryrun):
 726                 self.open(self.host, self.username, None, "Enter user name:")
 727                 self.sendPassword(self.password, "Enter Password:")
 728
 729                 #self.ifThenSend("RPC-16>", "Status")
 730                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 731
 732                 # Reboot Outlet  N        (Y/N)?
 733                 if dryrun:
 734                         self.ifThenSend("(Y/N)?", "N")
 735                 else:
 736                         self.ifThenSend("(Y/N)?", "Y")
 737                 self.ifThenSend("RPC3-NC>", "")
 738
 739                 self.close()
 740                 return 0
 741
 742 class BayTechRPC16(PCUControl):
 743         def run(self, node_port, dryrun):
 744                 self.open(self.host, self.username, None, "Enter user name:")
 745                 self.sendPassword(self.password, "Enter Password:")
 746
 747                 #self.ifThenSend("RPC-16>", "Status")
 748
 749                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 750
 751                 # Reboot Outlet  N        (Y/N)?
 752                 if dryrun:
 753                         self.ifThenSend("(Y/N)?", "N")
 754                 else:
 755                         self.ifThenSend("(Y/N)?", "Y")
 756                 self.ifThenSend("RPC-16>", "")
 757
 758                 self.close()
 759                 return 0
 760
 761 class BayTechCtrlCUnibe(PCUControl):
 762         """
 763                 For some reason, these units let you log in fine, but they hang
 764                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 765                 why.
 766         """
 767         def run(self, node_port, dryrun):
 768                 print "BayTechCtrlC %s" % self.host
 769
 770                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 771                 s = pxssh.pxssh()
 772                 if not s.login(self.host, self.username, self.password, ssh_options):
 773                         raise ExceptionPassword("Invalid Password")
 774                 # Otherwise, the login succeeded.
 775
 776                 # Send a ctrl-c to the remote process.
 777                 print "sending ctrl-c"
 778                 s.send(chr(3))
 779
 780                 # Control Outlets  (5 ,1).........5
 781                 try:
 782                         #index = s.expect("Enter Request")
 783                         index = s.expect(["Enter Request :"])
 784
 785                         if index == 0:
 786                                 print "3"
 787                                 s.send("3\r\n")
 788                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 789                                 if index == 1:
 790                                         s.send(self.username + "\r\n")
 791                                         index = s.expect(["DS-RPC>"])
 792
 793                                 if index == 0:
 794                                         print "Reboot %d" % node_port
 795                                         time.sleep(5)
 796                                         s.send("Reboot %d\r\n" % node_port)
 797
 798                                         time.sleep(5)
 799                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 800                                         if index == 0:
 801                                                 if dryrun:
 802                                                         print "sending N"
 803                                                         s.send("N\r\n")
 804                                                 else:
 805                                                         print "sending Y"
 806                                                         s.send("Y\r\n")
 807                                         elif index == 1:
 808                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 809                                         elif index == 2:
 810                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 811
 812                                 time.sleep(5)
 813                                 index = s.expect(["DS-RPC>"])
 814                                 #print "got prompt back"
 815
 816                         s.close()
 817
 818                 except pexpect.EOF:
 819                         raise ExceptionPrompt("EOF before expected Prompt")
 820                 except pexpect.TIMEOUT:
 821                         raise ExceptionPrompt("Timeout before expected Prompt")
 822
 823                 return 0
 824
 825 class BayTechCtrlC(PCUControl):
 826         """
 827                 For some reason, these units let you log in fine, but they hang
 828                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 829                 why.
 830         """
 831         def run(self, node_port, dryrun):
 832                 print "BayTechCtrlC %s" % self.host
 833
 834                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 835                 s = pxssh.pxssh()
 836                 if not s.login(self.host, self.username, self.password, ssh_options):
 837                         raise ExceptionPassword("Invalid Password")
 838                 # Otherwise, the login succeeded.
 839
 840                 # Send a ctrl-c to the remote process.
 841                 print "SENDING ctrl-c"
 842                 s.send(chr(3))
 843
 844                 # Control Outlets  (5 ,1).........5
 845                 try:
 846                         print "EXPECTING: ", "Enter Request :"
 847                         index = s.expect(["Enter Request :"])
 848
 849                         if index == 0:
 850                                 print "SENDING: 5"
 851                                 s.send("5\r\n")
 852                                 print "EXPECTING: ", "DS-RPC>"
 853                                 index = s.expect(["DS-RPC>", "Enter user name:", "Port in use."])
 854                                 if index == 1:
 855                                         print "sending username"
 856                                         s.send(self.username + "\r\n")
 857                                         index = s.expect(["DS-RPC>"])
 858                                 elif index == 2:
 859                                         raise ExceptionPrompt("PCU Reported 'Port in use.'")
 860
 861                                 if index == 0:
 862                                         print "SENDING: Reboot %d" % node_port
 863                                         s.send("Reboot %d\r\n" % node_port)
 864
 865                                         print "SLEEPING: 5"
 866                                         time.sleep(5)
 867                                         print "EXPECTING: ", "Y/N?"
 868                                         index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
 869                                         if index == 0:
 870                                                 if dryrun:
 871                                                         print "sending N"
 872                                                         s.send("N\r\n")
 873                                                 else:
 874                                                         print "SENDING: Y"
 875                                                         s.send("Y\r\n")
 876                                         elif index == 1:
 877                                                 raise ExceptionPrompt("PCU Reported 'Port in use.'")
 878                                         elif index == 2:
 879                                                 raise ExceptionSequence("Issued command 'Reboot' failed.")
 880
 881                                 # NOTE: for some reason, the script times out with the
 882                                 # following line.  In manual tests, it works correctly, but
 883                                 # with automated tests, evidently it fails.
 884                                 print "SLEEPING: 5"
 885                                 time.sleep(5)
 886                                 #print "TOTAL--", s.allstr, "--EOT"
 887                                 index = s.expect(["DS-RPC>"])
 888                                 print "got prompt back"
 889
 890                         s.close()
 891
 892                 except pexpect.EOF:
 893                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 894                 except pexpect.TIMEOUT:
 895                         raise ExceptionPrompt("Timeout before Prompt")
 896
 897                 return 0
 898
 899 class BayTech(PCUControl):
 900         supported_ports = [22,23]
 901         def run(self, node_port, dryrun):
 902                 self.open(self.host, self.username)
 903                 self.sendPassword(self.password)
 904
 905                 # Control Outlets  (5 ,1).........5
 906                 self.ifThenSend("Enter Request :", "5")
 907
 908                 # Reboot N
 909                 try:
 910                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 911                 except ExceptionNotFound, msg:
 912                         # one machine is configured to ask for a username,
 913                         # even after login...
 914                         print "msg: %s" % msg
 915                         self.transport.write(self.username + "\r\n")
 916                         time.sleep(5)
 917                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 918
 919                 # Reboot Outlet  N        (Y/N)?
 920                 if dryrun:
 921                         self.ifThenSend("(Y/N)?", "N")
 922                 else:
 923                         self.ifThenSend("(Y/N)?", "Y")
 924                 time.sleep(5)
 925                 self.ifThenSend("DS-RPC>", "")
 926
 927                 self.close()
 928                 return 0
 929
 930 class WTIIPS4(PCUControl):
 931         supported_ports = [23]
 932         def run(self, node_port, dryrun):
 933                 self.open(self.host)
 934                 self.sendPassword(self.password, "Enter Password:")
 935
 936                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 937                 if not dryrun:
 938                         self.ifThenSend("Sure? (Y/N): ", "N")
 939                 else:
 940                         self.ifThenSend("Sure? (Y/N): ", "Y")
 941
 942                 self.ifThenSend("IPS> ", "")
 943
 944                 self.close()
 945                 return 0
 946
 947 class ePowerSwitchNew(PCUControl):
 948         # NOTE:
 949         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
 950         #               For some reason this both doesn't work and in some cases, actually
 951         #               hangs the PCU.  Definitely not what we want.
 952         #
 953         #               The code below is much simpler.  Just letting things fail first,
 954         #               and then, trying again with authentication string in the header.
 955         #
 956         def run(self, node_port, dryrun):
 957                 self.transport = None
 958                 self.url = "http://%s:%d/" % (self.host,80)
 959                 uri = "%s:%d" % (self.host,80)
 960
 961                 req = urllib2.Request(self.url)
 962                 try:
 963                         handle = urllib2.urlopen(req)
 964                 except IOError, e:
 965                         # NOTE: this is expected to fail initially
 966                         pass
 967                 else:
 968                         print self.url
 969                         print "-----------"
 970                         print handle.read()
 971                         print "-----------"
 972                         return "ERROR: not protected by HTTP authentication"
 973
 974                 if not hasattr(e, 'code') or e.code != 401:
 975                         return "ERROR: failed for: %s" % str(e)
 976
 977                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
 978                 # NOTE: assuming basic realm authentication.
 979                 authheader = "Basic %s" % base64data
 980                 req.add_header("Authorization", authheader)
 981
 982                 try:
 983                         f = urllib2.urlopen(req)
 984                 except IOError, e:
 985                         # failing here means the User/passwd is wrong (hopefully)
 986                         raise ExceptionPassword("Incorrect username/password")
 987
 988                 # NOTE: after verifying that the user/password is correct,
 989                 #               actually reboot the given node.
 990                 if not dryrun:
 991                         try:
 992                                 data = urllib.urlencode({'P%d' % node_port : "r"})
 993                                 req = urllib2.Request(self.url + "cmd.html")
 994                                 req.add_header("Authorization", authheader)
 995                                 # add data to handler,
 996                                 f = urllib2.urlopen(req, data)
 997                                 if self.verbose: print f.read()
 998                         except:
 999                                 import traceback; traceback.print_exc()
1000
1001                                 # fetch url one more time on cmd.html, econtrol.html or whatever.
1002                                 # pass
1003                 else:
1004                         if self.verbose: print f.read()
1005
1006                 self.close()
1007                 return 0
1008
1009 class CustomPCU(PCUControl):
1010         def run(self, node_port, dryrun):
1011                 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
1012
1013                 if not dryrun:
1014                         # Turn host off, then on
1015                         formstr = "plab%s=off" % node_port
1016                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
1017                         time.sleep(5)
1018                         formstr = "plab%s=on" % node_port
1019                         os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
1020                 else:
1021                         os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
1022
1023
1024 class ePowerSwitchOld(PCUControl):
1025         def run(self, node_port, dryrun):
1026                 self.url = "http://%s:%d/" % (self.host,80)
1027                 uri = "%s:%d" % (self.host,80)
1028
1029                 # create authinfo
1030                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1031                 authinfo.add_password (None, uri, self.username, self.password)
1032                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1033
1034                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
1035                 transport = urllib2.build_opener(authinfo)
1036                 f = transport.open(self.url)
1037                 if self.verbose: print f.read()
1038
1039                 if not dryrun:
1040                         transport = urllib2.build_opener(authhandler)
1041                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
1042                         if self.verbose: print f.read()
1043
1044                 self.close()
1045                 return 0
1046
1047 class ePowerSwitchOld(PCUControl):
1048         supported_ports = [80]
1049         def run(self, node_port, dryrun):
1050                 self.url = "http://%s:%d/" % (self.host,80)
1051                 uri = "%s:%d" % (self.host,80)
1052
1053                 # TODO: I'm still not sure what the deal is here.
1054                 #               two independent calls appear to need to be made before the
1055                 #               reboot will succeed.  It doesn't seem to be possible to do
1056                 #               this with a single call.  I have no idea why.
1057
1058                 # create authinfo
1059                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1060                 authinfo.add_password (None, uri, self.username, self.password)
1061                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1062
1063                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
1064                 transport = urllib2.build_opener()
1065                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
1066                 if self.verbose: print f.read()
1067
1068                 if not dryrun:
1069                         transport = urllib2.build_opener(authhandler)
1070                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
1071                         if self.verbose: print f.read()
1072
1073                 #       data= "P%d=r" % node_port
1074                 #self.open(self.host, self.username, self.password)
1075                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
1076                 #self.sendHTTP("econtrol.html", data)
1077                 #self.sendHTTP("cmd.html", data)
1078
1079                 self.close()
1080                 return 0
1081
1082 class ManualPCU(PCUControl):
1083         supported_ports = [22,23,80,443,9100,16992]
1084
1085         def run(self, node_port, dryrun):
1086                 if not dryrun:
1087                         # TODO: send email message to monitor admin requesting manual
1088                         # intervention.  This should always be an option for ridiculous,
1089                         # custom jobs.
1090                         pass
1091                 return 0
1092
1093 class PM211MIP(ManualPCU):
1094         supported_ports = [80,443]
1095
1096 ### rebooting european BlackBox PSE boxes
1097 # Thierry Parmentelat - May 11 2005
1098 # tested on 4-ports models known as PSE505-FR
1099 # uses http to POST a data 'P<port>=r'
1100 # relies on basic authentication within http1.0
1101 # first curl-based script was
1102 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
1103 #       http://<hostname>:<http_port>/cmd.html && echo OK
1104
1105 # log in:
1106
1107 ## BB PSMaverick
1108 class BlackBoxPSMaverick(PCUControl):
1109         supported_ports = [80]
1110
1111         def run(self, node_port, dryrun):
1112                 if not dryrun:
1113                         # send reboot signal.
1114                         cmd = "curl -s --data 'P%s=r' --anyauth --user '%s:%s' http://%s/config/home_f.html" % ( node_port, self.username, self.password, self.host)
1115                 else:
1116                         # else, just try to log in
1117                         cmd = "curl -s --anyauth --user '%s:%s' http://%s/config/home_f.html" % ( self.username, self.password, self.host)
1118
1119                 p = os.popen(cmd)
1120                 result = p.read()
1121                 print "RESULT: ", result
1122
1123                 if len(result.split()) > 3:
1124                         return 0
1125                 else:
1126                         return result
1127
1128 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
1129
1130         global verbose
1131
1132         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
1133         data= "P%d=r" % port_in_pcu
1134         if verbose:
1135                 logger.debug("POSTing '%s' on %s" % (data,url))
1136
1137         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1138         uri = "%s:%d" % (pcu_ip,http_port)
1139         authinfo.add_password (None, uri, username, password)
1140         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1141
1142         opener = urllib2.build_opener(authhandler)
1143         urllib2.install_opener(opener)
1144
1145         if (dryrun):
1146                 return 0
1147
1148         try:
1149                 f = urllib2.urlopen(url,data)
1150
1151                 r= f.read()
1152                 if verbose:
1153                         logger.debug(r)
1154                 return 0
1155
1156         except urllib2.URLError,err:
1157                 logger.info('Could not open http connection', err)
1158                 return "bbpse error"
1159
1160 ### rebooting x10toggle based systems addressed by port
1161 # Marc E. Fiuczynski - May 31 2005
1162 # tested on 4-ports models known as PSE505-FR
1163 # uses ssh and password to login to an account
1164 # that will cause the system to be powercycled.
1165
1166 def x10toggle_reboot(ip, username, password, port, dryrun):
1167         global verbose
1168
1169         ssh = None
1170         try:
1171                 ssh = pyssh.Ssh(username, ip)
1172                 ssh.open()
1173
1174                 # Login
1175                 telnet_answer(ssh, "password:", password)
1176
1177                 if not dryrun:
1178                         # Reboot
1179                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
1180
1181                 # Close
1182                 output = ssh.close()
1183                 if verbose:
1184                         logger.debug(output)
1185                 return 0
1186
1187         except Exception, err:
1188                 if verbose:
1189                         logger.debug(err)
1190                 if ssh:
1191                         output = ssh.close()
1192                         if verbose:
1193                                 logger.debug(output)
1194                 return errno.ETIMEDOUT
1195
1196 ### rebooting Dell systems via RAC card
1197 # Marc E. Fiuczynski - June 01 2005
1198 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1199 #
1200
1201 def runcmd(command, args, username, password, timeout = None):
1202
1203         result = [None]
1204         result_ready = threading.Condition()
1205
1206         def set_result(x):
1207
1208                 result_ready.acquire()
1209                 try:
1210                         result[0] = x
1211                 finally:
1212                         result_ready.notify()
1213                         result_ready.release()
1214
1215         def do_command(command, username, password):
1216
1217                 try:
1218                         # Popen4 is a popen-type class that combines stdout and stderr
1219                         p = popen2.Popen4(command)
1220
1221                         # read all output data
1222                         p.tochild.write("%s\n" % username)
1223                         p.tochild.write("%s\n" % password)
1224                         p.tochild.close()
1225                         data = p.fromchild.read()
1226
1227                         while True:
1228                                 # might get interrupted by a signal in poll() or waitpid()
1229                                 try:
1230                                         retval = p.wait()
1231                                         set_result((retval, data))
1232                                         break
1233                                 except OSError, ex:
1234                                         if ex.errno == errno.EINTR:
1235                                                 continue
1236                                         raise ex
1237                 except Exception, ex:
1238                         set_result(ex)
1239
1240         if args:
1241                 command = " ".join([command] + args)
1242
1243         worker = threading.Thread(target = do_command, args = (command, username, password, ))
1244         worker.setDaemon(True)
1245         result_ready.acquire()
1246         worker.start()
1247         result_ready.wait(timeout)
1248         try:
1249                 if result == [None]:
1250                         raise Exception, "command timed-out: '%s'" % command
1251         finally:
1252                 result_ready.release()
1253         result = result[0]
1254
1255         if isinstance(result, Exception):
1256                 raise result
1257         else:
1258                 (retval, data) = result
1259                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1260                         return data
1261                 else:
1262                         out = "system command ('%s') " % command
1263                         if os.WIFEXITED(retval):
1264                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1265                         else:
1266                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1267                         if data:
1268                                 out += "; output follows:\n" + data
1269                         raise Exception, out
1270
1271 def racadm_reboot(host, username, password, port, dryrun):
1272         global verbose
1273
1274         ip = socket.gethostbyname(host)
1275         try:
1276                 cmd = "/usr/sbin/racadm"
1277                 os.stat(cmd)
1278                 if not dryrun:
1279                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1280                                 username, password)
1281                 else:
1282                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1283                                 username, password)
1284
1285                 print "RUNCMD: %s" % output
1286                 if verbose:
1287                         logger.debug(output)
1288                 return 0
1289
1290         except Exception, err:
1291                 logger.debug("runcmd raised exception %s" % err)
1292                 if verbose:
1293                         logger.debug(err)
1294                 return -1
1295
1296 def pcu_name(pcu):
1297         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1298                 return pcu['hostname']
1299         elif pcu['ip'] is not None and pcu['ip'] is not "":
1300                 return pcu['ip']
1301         else:
1302                 return None
1303
1304
1305 def get_pcu_values(pcu_id):
1306         print "pcuid: %s" % pcu_id
1307         try:
1308                 pcurec = FindbadPCURecord.get_latest_by(plc_pcuid=pcu_id).first()
1309                 if pcurec:
1310                         values = pcurec.to_dict()
1311                 else:
1312                         values = None
1313         except:
1314                 values = None
1315
1316         return values
1317
1318 def reboot(nodename):
1319         return reboot_policy(nodename, True, False)
1320
1321 def reboot_str(nodename):
1322         global verbose
1323         continue_probe = True
1324         dryrun=False
1325
1326         pcu = plc.getpcu(nodename)
1327         if not pcu:
1328                 logger.debug("no pcu for %s" % nodename)
1329                 print "no pcu for %s" % nodename
1330                 return False # "%s has no pcu" % nodename
1331
1332         values = get_pcu_values(pcu['pcu_id'])
1333         if values == None:
1334                 logger.debug("No values for pcu probe %s" % nodename)
1335                 print "No values for pcu probe %s" % nodename
1336                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1337
1338         # Try the PCU first
1339         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1340
1341         ret = reboot_test_new(nodename, values, verbose, dryrun)
1342         return ret
1343
1344 def reboot_policy(nodename, continue_probe, dryrun):
1345         global verbose
1346
1347         pcu = plc.getpcu(nodename)
1348         if not pcu:
1349                 logger.debug("no pcu for %s" % nodename)
1350                 print "no pcu for %s" % nodename
1351                 return False # "%s has no pcu" % nodename
1352
1353         values = get_pcu_values(pcu['pcu_id'])
1354         if values == None:
1355                 logger.debug("No values for pcu probe %s" % nodename)
1356                 print "No values for pcu probe %s" % nodename
1357                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1358
1359         # Try the PCU first
1360         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1361
1362         ret = reboot_test_new(nodename, values, verbose, dryrun)
1363
1364         if ret != 0:
1365                 print ret
1366                 return False
1367         else:
1368                 print "return true"
1369                 return True
1370
1371 class Unknown(PCUControl):
1372         supported_ports = [22,23,80,443,5869,9100,16992]
1373
1374 def model_to_object(modelname):
1375         if "AMT" in modelname:
1376                 return IntelAMT
1377         elif "BayTech" in modelname:
1378                 return BayTech
1379         elif "HPiLO" in modelname:
1380                 return HPiLO
1381         elif "IPAL" in modelname:
1382                 return IPAL
1383         elif "APC" in modelname:
1384                 return APCControl
1385         elif "DRAC" in modelname:
1386                 return DRAC
1387         elif "WTI" in modelname:
1388                 return WTIIPS4
1389         elif "ePowerSwitch" in modelname:
1390                 return ePowerSwitchNew
1391         elif "IPMI" in modelname:
1392                 return IPMI
1393         elif "BlackBoxPSMaverick" in modelname:
1394                 return BlackBoxPSMaverick
1395         elif "PM211MIP" in modelname:
1396                 return PM211MIP
1397         elif "ManualPCU" in modelname:
1398                 return ManualPCU
1399         else:
1400                 print "UNKNOWN model %s"%modelname
1401                 return Unknown
1402
1403 def reboot_test_new(nodename, values, verbose, dryrun):
1404         rb_ret = ""
1405         if 'plc_pcu_stats' in values:
1406                 values.update(values['plc_pcu_stats'])
1407
1408         try:
1409                 modelname = values['model']
1410                 if modelname:
1411                         object = eval('%s(values, verbose, ["22", "23", "80", "443", "9100", "16992", "5869"])' % modelname)
1412                         rb_ret = object.reboot(values[nodename], dryrun)
1413                 else:
1414                         rb_ret =  "Not_Run"
1415                 # TODO: how to handle the weird, georgetown pcus, the drac faults, and ilo faults
1416         except ExceptionPort, err:
1417                 rb_ret = str(err)
1418
1419         return rb_ret
1420
1421
1422 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1423         rb_ret = ""
1424         if 'plc_pcu_stats' in values:
1425                 values.update(values['plc_pcu_stats'])
1426
1427         try:
1428                 # DataProbe iPal (many sites)
1429                 if  continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1430                         ipal = IPAL(values, verbose, ['23', '80', '9100'])
1431                         rb_ret = ipal.reboot(values[nodename], dryrun)
1432
1433                 # APC Masterswitch (Berkeley)
1434                 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1435                                                                   values['model'].find("Masterswitch") >= 0 ):
1436                         print values
1437
1438                         # TODO: make a more robust version of APC
1439                         if values['pcu_id'] in [1102,1163,1055,1111,1231,1113,1127,1128,1148]:
1440                                 apc = APCControl12p3(values, verbose, ['22', '23'])
1441                                 rb_ret = apc.reboot(values[nodename], dryrun)
1442
1443                         elif values['pcu_id'] in [1110,86]:
1444                                 apc = APCControl1p4(values, verbose, ['22', '23'])
1445                                 rb_ret = apc.reboot(values[nodename], dryrun)
1446
1447                         elif values['pcu_id'] in [1221,1225,1220,1192]:
1448                                 apc = APCControl121p3(values, verbose, ['22', '23'])
1449                                 rb_ret = apc.reboot(values[nodename], dryrun)
1450
1451                         elif values['pcu_id'] in [1173,1240,47,1363,1405,1401,1372,1371]:
1452                                 apc = APCControl121p1(values, verbose, ['22', '23'])
1453                                 rb_ret = apc.reboot(values[nodename], dryrun)
1454
1455                         else:
1456                                 apc = APCControl13p13(values, verbose, ['22', '23'])
1457                                 rb_ret = apc.reboot(values[nodename], dryrun)
1458
1459                 # BayTech DS4-RPC
1460                 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1461                         if values['pcu_id'] in [1056,1237,1052,1209,1002,1008,1041,1013,1022]:
1462                                 # These  require a 'ctrl-c' to be sent...
1463                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1464                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1465
1466                         elif values['pcu_id'] in [93]:
1467                                 baytech = BayTechRPC3NC(values, verbose, ['22', '23'])
1468                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1469
1470                         elif values['pcu_id'] in [1057]:
1471                                 # These  require a 'ctrl-c' to be sent...
1472                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1473                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1474
1475                         elif values['pcu_id'] in [1012]:
1476                                 # This pcu sometimes doesn't present the 'Username' prompt,
1477                                 # unless you immediately try again...
1478                                 try:
1479                                         baytech = BayTechRPC16(values, verbose, ['22', '23'])
1480                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1481                                 except:
1482                                         baytech = BayTechRPC16(values, verbose, ['22', '23'])
1483                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1484                         else:
1485                                 baytech = BayTech(values, verbose, ['22', '23'])
1486                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1487
1488                 # iLO
1489                 elif continue_probe and values['model'].find("ilo") >= 0:
1490                         try:
1491                                 hpilo = HPiLO(values, verbose, ['22'])
1492                                 rb_ret = hpilo.reboot(0, dryrun)
1493                                 if rb_ret != 0:
1494                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1495                                         rb_ret = hpilo.reboot(0, dryrun)
1496                         except:
1497                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1498                                 rb_ret = hpilo.reboot(0, dryrun)
1499
1500                 # DRAC ssh
1501                 elif continue_probe and values['model'].find("DRAC") >= 0:
1502                         # TODO: I don't think DRAC will throw an exception for the
1503                         # default method to catch...
1504                         try:
1505                                 drac = DRAC(values, verbose, ['443', '5869'])
1506                                 rb_ret = drac.reboot(0, dryrun)
1507                         except:
1508                                 drac = DRACDefault(values, verbose, ['22'])
1509                                 rb_ret = drac.reboot(0, dryrun)
1510
1511                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1512                                 wti = WTIIPS4(values, verbose, ['23'])
1513                                 rb_ret = wti.reboot(values[nodename], dryrun)
1514
1515                 elif continue_probe and values['model'].find("AMT") >= 0:
1516                                 amt = IntelAMT(values, verbose, ['16992'])
1517                                 rb_ret = amt.reboot(values[nodename], dryrun)
1518
1519                 elif continue_probe and values['model'].find("bbsemaverick") >=0:
1520                         print "TRYING BlackBoxPSMaverick"
1521                         bbe = BlackBoxPSMaverick(values, verbose, ['80'])
1522                         rb_ret = bbe.reboot(values[nodename], dryrun)
1523
1524                 elif continue_probe and values['model'].find("ipmi") >=0:
1525
1526                         print "TRYING IPMI"
1527                         ipmi = IPMI(values, verbose, ['80', '443', '623'])
1528                         rb_ret = ipmi.reboot(values[nodename], dryrun)
1529
1530                 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1531                         # TODO: allow a different port than http 80.
1532                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1533                                 eps = ePowerSwitchNew(values, verbose, ['80'])
1534                         elif values['pcu_id'] in [1003]:
1535                                 # OLD EPOWER
1536                                 print "OLD EPOWER"
1537                                 eps = ePowerSwitchOld(values, verbose, ['80'])
1538                         else:
1539                                 eps = ePowerSwitchNew(values, verbose, ['80'])
1540
1541                         rb_ret = eps.reboot(values[nodename], dryrun)
1542                 elif continue_probe and values['pcu_id'] in [1122]:
1543                         custom = CustomPCU(values, verbose, ['80', '443'])
1544                         custom.reboot(values[nodename], dryrun)
1545
1546                 elif continue_probe:
1547                         rb_ret = "Unsupported_PCU"
1548
1549                 elif continue_probe == False:
1550                         if 'port_status' in values:
1551                                 rb_ret = "NetDown"
1552                         else:
1553                                 rb_ret = "Not_Run"
1554                 else:
1555                         rb_ret = -1
1556
1557         except ExceptionPort, err:
1558                 rb_ret = str(err)
1559
1560         return rb_ret
1561         # ????
1562         #elif continue_probe and values['protocol'] == "racadm" and \
1563         #               values['model'] == "RAC":
1564         #       rb_ret = racadm_reboot(pcu_name(values),
1565         #                                                                 values['username'],
1566         #                                                                 values['password'],
1567         #                                                                 pcu[nodename],
1568         #                                                                 dryrun)
1569
1570 def main():
1571         logger.setLevel(logging.DEBUG)
1572         ch = logging.StreamHandler()
1573         ch.setLevel(logging.DEBUG)
1574         formatter = logging.Formatter('LOGGER - %(message)s')
1575         ch.setFormatter(formatter)
1576         logger.addHandler(ch)
1577
1578         try:
1579                 if "test" in sys.argv:
1580                         dryrun = True
1581                 else:
1582                         dryrun = False
1583
1584                 for node in sys.argv[1:]:
1585                         if node == "test": continue
1586
1587                         print "Rebooting %s" % node
1588                         if reboot_policy(node, True, dryrun):
1589                                 print "success"
1590                         else:
1591                                 print "failed"
1592         except Exception, err:
1593                 import traceback; traceback.print_exc()
1594                 print err
1595
1596 if __name__ == '__main__':
1597         logger = logging.getLogger("monitor")
1598         main()