reboot.py

   1 #!/usr/bin/python
   2 #
   3 # Reboot specified nodes
   4 #
   5
   6 import getpass, getopt
   7 import os, sys
   8 import xml, xmlrpclib
   9 import errno, time, traceback
  10 import urllib2
  11 import threading, popen2
  12 import array, struct
  13 #from socket import *
  14 import socket
  15 import plc
  16 import base64
  17 from subprocess import PIPE, Popen
  18 import ssh.pxssh as pxssh
  19 import ssh.pexpect as pexpect
  20 import socket
  21
  22 # Use our versions of telnetlib and pyssh
  23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
  24 import telnetlib
  25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
  26 import pyssh
  27
  28 # Timeouts in seconds
  29 TELNET_TIMEOUT = 45
  30
  31 # Event class ID from pcu events
  32 #NODE_POWER_CONTROL = 3
  33
  34 # Monitor user ID
  35 #MONITOR_USER_ID = 11142
  36
  37 import logging
  38 logger = logging.getLogger("monitor")
  39 verbose = 1
  40 #dryrun = 0;
  41
  42 class ExceptionNoTransport(Exception): pass
  43 class ExceptionNotFound(Exception): pass
  44 class ExceptionPassword(Exception): pass
  45 class ExceptionTimeout(Exception): pass
  46 class ExceptionPrompt(Exception): pass
  47 class ExceptionSequence(Exception): pass
  48 class ExceptionReset(Exception): pass
  49 class ExceptionPort(Exception): pass
  50 class ExceptionUsername(Exception): pass
  51
  52 def telnet_answer(telnet, expected, buffer):
  53         global verbose
  54
  55         output = telnet.read_until(expected, TELNET_TIMEOUT)
  56         #if verbose:
  57         #       logger.debug(output)
  58         if output.find(expected) == -1:
  59                 raise ExceptionNotFound, "'%s' not found" % expected
  60         else:
  61                 telnet.write(buffer + "\r\n")
  62
  63
  64 # PCU has model, host, preferred-port, user, passwd,
  65
  66 # This is an object derived directly form the PLCAPI DB fields
  67 class PCU(object):
  68         def __init__(self, plc_pcu_dict):
  69                 for field in ['username', 'password', 'site_id',
  70                                                 'hostname', 'ip',
  71                                                 'pcu_id', 'model',
  72                                                 'node_ids', 'ports', ]:
  73                         if field in plc_pcu_dict:
  74                                 self.__setattr__(field, plc_pcu_dict[field])
  75                         else:
  76                                 raise Exception("No such field %s in PCU object" % field)
  77
  78 # These are the convenience functions build around the PCU object.
  79 class PCUModel(PCU):
  80         def __init__(self, plc_pcu_dict):
  81                 PCU.__init__(self, plc_pcu_dict)
  82                 self.host = self.pcu_name()
  83
  84         def pcu_name(self):
  85                 if self.hostname is not None and self.hostname is not "":
  86                         return self.hostname
  87                 elif self.ip is not None and self.ip is not "":
  88                         return self.ip
  89                 else:
  90                         return None
  91
  92         def nodeidToPort(self, node_id):
  93                 if node_id in self.node_ids:
  94                         for i in range(0, len(self.node_ids)):
  95                                 if node_id == self.node_ids[i]:
  96                                         return self.ports[i]
  97
  98                 raise Exception("No such Node ID: %d" % node_id)
  99
 100 # This class captures the observed pcu records from FindBadPCUs.py
 101 class PCURecord:
 102         def __init__(self, pcu_record_dict):
 103                 for field in ['nodenames', 'portstatus',
 104                                                 'dnsmatch',
 105                                                 'complete_entry', ]:
 106                         if field in pcu_record_dict:
 107                                 if field == "reboot":
 108                                         self.__setattr__("reboot_str", pcu_record_dict[field])
 109                                 else:
 110                                         self.__setattr__(field, pcu_record_dict[field])
 111                         else:
 112                                 raise Exception("No such field %s in pcu record dict" % field)
 113
 114 class Transport:
 115         TELNET = 1
 116         SSH    = 2
 117         HTTP   = 3
 118
 119         TELNET_TIMEOUT = 60
 120
 121         def __init__(self, type, verbose):
 122                 self.type = type
 123                 self.verbose = verbose
 124                 self.transport = None
 125
 126 #       def __del__(self):
 127 #               if self.transport:
 128 #                       self.close()
 129
 130         def open(self, host, username=None, password=None, prompt="User Name"):
 131                 transport = None
 132
 133                 if self.type == self.TELNET:
 134                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
 135                         transport.set_debuglevel(self.verbose)
 136                         if username is not None:
 137                                 self.transport = transport
 138                                 self.ifThenSend(prompt, username, ExceptionUsername)
 139
 140                 elif self.type == self.SSH:
 141                         if username is not None:
 142                                 transport = pyssh.Ssh(username, host)
 143                                 transport.set_debuglevel(self.verbose)
 144                                 transport.open()
 145                                 # TODO: have an ssh set_debuglevel() also...
 146                         else:
 147                                 raise Exception("Username cannot be None for ssh transport.")
 148                 elif self.type == self.HTTP:
 149                         self.url = "http://%s:%d/" % (host,80)
 150                         uri = "%s:%d" % (host,80)
 151
 152                         # create authinfo
 153                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 154                         authinfo.add_password (None, uri, username, password)
 155                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 156
 157                         transport = urllib2.build_opener(authhandler)
 158
 159                 else:
 160                         raise Exception("Unknown transport type: %s" % self.type)
 161
 162                 self.transport = transport
 163                 return True
 164
 165         def close(self):
 166                 if self.type == self.TELNET:
 167                         self.transport.close()
 168                 elif self.type == self.SSH:
 169                         self.transport.close()
 170                 elif self.type == self.HTTP:
 171                         pass
 172                 else:
 173                         raise Exception("Unknown transport type %s" % self.type)
 174                 self.transport = None
 175
 176         def sendHTTP(self, resource, data):
 177                 if self.verbose:
 178                         print "POSTing '%s' to %s" % (data,self.url + resource)
 179
 180                 try:
 181                         f = self.transport.open(self.url + resource ,data)
 182                         r = f.read()
 183                         if self.verbose:
 184                                 print r
 185
 186                 except urllib2.URLError,err:
 187                         logger.info('Could not open http connection', err)
 188                         return "http transport error"
 189
 190                 return 0
 191
 192         def sendPassword(self, password, prompt=None):
 193                 if self.type == self.TELNET:
 194                         if prompt == None:
 195                                 self.ifThenSend("Password", password, ExceptionPassword)
 196                         else:
 197                                 self.ifThenSend(prompt, password, ExceptionPassword)
 198                 elif self.type == self.SSH:
 199                         self.ifThenSend("password:", password, ExceptionPassword)
 200                 elif self.type == self.HTTP:
 201                         pass
 202                 else:
 203                         raise Exception("Unknown transport type: %s" % self.type)
 204
 205         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
 206
 207                 if self.transport != None:
 208                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
 209                         if output.find(expected) == -1:
 210                                 raise ErrorClass, "'%s' not found" % expected
 211                         else:
 212                                 self.transport.write(buffer + "\r\n")
 213                 else:
 214                         raise ExceptionNoTransport("transport object is type None")
 215
 216         def ifElse(self, expected, ErrorClass):
 217                 try:
 218                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
 219                 except:
 220                         raise ErrorClass("Could not find '%s' within timeout" % expected)
 221
 222
 223 class PCUControl(Transport,PCUModel,PCURecord):
 224         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
 225                 PCUModel.__init__(self, plc_pcu_record)
 226                 PCURecord.__init__(self, plc_pcu_record)
 227                 type = None
 228                 if self.portstatus:
 229                         if '22' in supported_ports and self.portstatus['22'] == "open":
 230                                 type = Transport.SSH
 231                         elif '23' in supported_ports and self.portstatus['23'] == "open":
 232                                 type = Transport.TELNET
 233                         elif '80' in supported_ports and self.portstatus['80'] == "open":
 234                                 type = Transport.HTTP
 235                         elif '443' in supported_ports and self.portstatus['443'] == "open":
 236                                 type = Transport.HTTP
 237                         elif '5869' in supported_ports and self.portstatus['5869'] == "open":
 238                                 # For DRAC cards.  not sure how much it's used in the
 239                                 # protocol.. but racadm opens this port.
 240                                 type = Transport.HTTP
 241                         else:
 242                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
 243                 else:
 244                         raise Exception("No Portstatus: No transport because no open ports")
 245                 Transport.__init__(self, type, verbose)
 246
 247         def run(self, node_port, dryrun):
 248                 """ This function is to be defined by the specific PCU instance.  """
 249                 pass
 250
 251         def reboot(self, node_port, dryrun):
 252                 try:
 253                         return self.run(node_port, dryrun)
 254                 except ExceptionNotFound, err:
 255                         return "error: " + str(err)
 256                 except ExceptionPassword, err:
 257                         return "password exception: " + str(err)
 258                 except ExceptionTimeout, err:
 259                         return "timeout exception: " + str(err)
 260                 except ExceptionUsername, err:
 261                         return "exception: no username prompt: " + str(err)
 262                 except ExceptionSequence, err:
 263                         return "sequence error: " + str(err)
 264                 except ExceptionPrompt, err:
 265                         return "prompt exception: " + str(err)
 266                 except ExceptionPort, err:
 267                         return "no ports exception: " + str(err)
 268                 except socket.error, err:
 269                         return "socket error: timeout: " + str(err)
 270                 except EOFError, err:
 271                         if self.verbose:
 272                                 logger.debug("reboot: EOF")
 273                                 logger.debug(err)
 274                         self.transport.close()
 275                         import traceback
 276                         traceback.print_exc()
 277                         return "EOF connection reset" + str(err)
 278                 #except Exception, err:
 279                 #       if self.verbose:
 280                 #               logger.debug("reboot: Exception")
 281                 #               logger.debug(err)
 282                 #       if self.transport:
 283                 #               self.transport.close()
 284                 #       import traceback
 285                 #       traceback.print_exc()
 286                 #       return  "generic exception; unknown problem."
 287
 288
 289 class IPAL(PCUControl):
 290         def run(self, node_port, dryrun):
 291                 self.open(self.host)
 292
 293                 # XXX Some iPals require you to hit Enter a few times first
 294                 self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
 295
 296                 # Login
 297                 self.ifThenSend("Password >", self.password, ExceptionPassword)
 298                 self.transport.write("\r\n\r\n")
 299
 300                 if not dryrun: # P# - Pulse relay
 301                         self.ifThenSend("Enter >",
 302                                                         "P%d" % node_port,
 303                                                         ExceptionNotFound)
 304                 # Get the next prompt
 305                 self.ifElse("Enter >", ExceptionTimeout)
 306
 307                 self.close()
 308                 return 0
 309
 310 class APCEurope(PCUControl):
 311         def run(self, node_port, dryrun):
 312                 self.open(self.host, self.username)
 313                 self.sendPassword(self.password)
 314
 315                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 316                 self.ifThenSend("\r\n> ", "2")
 317                 self.ifThenSend("\r\n> ", str(node_port))
 318                 # 3- Immediate Reboot
 319                 self.ifThenSend("\r\n> ", "3")
 320
 321                 if not dryrun:
 322                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 323                                                         "YES\r\n",
 324                                                         ExceptionSequence)
 325                 else:
 326                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 327                                                         "", ExceptionSequence)
 328                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 329
 330                 self.close()
 331                 return 0
 332
 333 class APCBrazil(PCUControl):
 334         def run(self, node_port, dryrun):
 335                 self.open(self.host, self.username)
 336                 self.sendPassword(self.password)
 337
 338                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 339                 self.ifThenSend("\r\n> ", str(node_port))
 340                 # 4- Immediate Reboot
 341                 self.ifThenSend("\r\n> ", "4")
 342
 343                 if not dryrun:
 344                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 345                                                         "YES\r\n",
 346                                                         ExceptionSequence)
 347                 else:
 348                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 349                                                         "", ExceptionSequence)
 350                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 351
 352                 self.close()
 353                 return 0
 354
 355 class APCBerlin(PCUControl):
 356         def run(self, node_port, dryrun):
 357                 self.open(self.host, self.username)
 358                 self.sendPassword(self.password)
 359
 360                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 361                 self.ifThenSend("\r\n> ", "2")
 362                 self.ifThenSend("\r\n> ", "1")
 363                 self.ifThenSend("\r\n> ", str(node_port))
 364                 # 3- Immediate Reboot
 365                 self.ifThenSend("\r\n> ", "3")
 366
 367                 if not dryrun:
 368                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 369                                                         "YES\r\n",
 370                                                         ExceptionSequence)
 371                 else:
 372                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 373                                                         "", ExceptionSequence)
 374                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 375
 376                 self.close()
 377                 return 0
 378
 379 class APCFolsom(PCUControl):
 380         def run(self, node_port, dryrun):
 381                 self.open(self.host, self.username)
 382                 self.sendPassword(self.password)
 383
 384                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 385                 self.ifThenSend("\r\n> ", "2")
 386                 self.ifThenSend("\r\n> ", "1")
 387                 self.ifThenSend("\r\n> ", str(node_port))
 388                 self.ifThenSend("\r\n> ", "1")
 389
 390                 # 3- Immediate Reboot
 391                 self.ifThenSend("\r\n> ", "3")
 392
 393                 if not dryrun:
 394                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 395                                                         "YES\r\n",
 396                                                         ExceptionSequence)
 397                 else:
 398                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 399                                                         "", ExceptionSequence)
 400                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 401
 402                 self.close()
 403                 return 0
 404
 405 class APCMaster(PCUControl):
 406         def run(self, node_port, dryrun):
 407                 self.open(self.host, self.username)
 408                 self.sendPassword(self.password)
 409
 410                 # 1- Device Manager
 411                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
 412                 # 3- Outlet Control/Config
 413                 self.ifThenSend("\r\n> ", "3")
 414                 # n- Outlet n
 415                 self.ifThenSend("\r\n> ", str(node_port))
 416                 # 1- Control Outlet
 417                 self.ifThenSend("\r\n> ", "1")
 418                 # 3- Immediate Reboot
 419                 self.ifThenSend("\r\n> ", "3")
 420
 421                 if not dryrun:
 422                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 423                                                         "YES\r\n",
 424                                                         ExceptionSequence)
 425                 else:
 426                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
 427                                                         "", ExceptionSequence)
 428                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
 429
 430                 self.close()
 431                 return 0
 432
 433 class APC(PCUControl):
 434         def __init__(self, plc_pcu_record, verbose):
 435                 PCUControl.__init__(self, plc_pcu_record, verbose)
 436
 437                 self.master = APCMaster(plc_pcu_record, verbose)
 438                 self.folsom = APCFolsom(plc_pcu_record, verbose)
 439                 self.europe = APCEurope(plc_pcu_record, verbose)
 440
 441         def run(self, node_port, dryrun):
 442                 try_again = True
 443                 sleep_time = 1
 444
 445                 for pcu in [self.master, self.europe, self.folsom]:
 446                         if try_again:
 447                                 try:
 448                                         print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
 449                                         try_again = False
 450                                         print "sleeping 5"
 451                                         time.sleep(sleep_time)
 452                                         ret = pcu.reboot(node_port, dryrun)
 453                                 except ExceptionSequence, err:
 454                                         del pcu
 455                                         sleep_time = 130
 456                                         try_again = True
 457
 458                 if try_again:
 459                         return "Unknown reboot sequence for APC PCU"
 460                 else:
 461                         return ret
 462
 463 class DRACRacAdm(PCUControl):
 464         def run(self, node_port, dryrun):
 465
 466                 print "trying racadm_reboot..."
 467                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
 468
 469                 return 0
 470
 471 class DRAC(PCUControl):
 472         def run(self, node_port, dryrun):
 473                 self.open(self.host, self.username)
 474                 self.sendPassword(self.password)
 475
 476                 print "logging in..."
 477                 self.transport.write("\r\n")
 478                 # Testing Reboot ?
 479                 if dryrun:
 480                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
 481                 else:
 482                         # Reset this machine
 483                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
 484
 485                 self.ifThenSend("[%s]#" % self.username, "exit")
 486
 487                 self.close()
 488                 return 0
 489
 490 class HPiLO(PCUControl):
 491         def run(self, node_port, dryrun):
 492                 self.open(self.host, self.username)
 493                 self.sendPassword(self.password)
 494
 495                 # </>hpiLO->
 496                 self.ifThenSend("</>hpiLO->", "cd system1")
 497
 498                 # Reboot Outlet  N        (Y/N)?
 499                 if dryrun:
 500                         self.ifThenSend("</system1>hpiLO->", "POWER")
 501                 else:
 502                         # Reset this machine
 503                         self.ifThenSend("</system1>hpiLO->", "reset")
 504
 505                 self.ifThenSend("</system1>hpiLO->", "exit")
 506
 507                 self.close()
 508                 return 0
 509
 510
 511 class HPiLOHttps(PCUControl):
 512         def run(self, node_port, dryrun):
 513                 import soltesz
 514
 515                 #cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s'" % (
 516                 #                       self.host, "iloxml/Get_Network.xml",
 517                 #                       self.username, self.password)
 518                 #p_ilo  = Popen(cmd, stdout=PIPE, shell=True)
 519                 #cmd2 = "grep 'MESSAGE' | grep -v 'No error'"
 520                 #p_grep = Popen(cmd2, stdin=p_ilo.stdout, stdout=PIPE, stderr=PIPE, shell=True)
 521                 #sout, serr = p_grep.communicate()
 522
 523                 locfg = soltesz.CMD()
 524                 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 525                                         self.host, "iloxml/Get_Network.xml",
 526                                         self.username, self.password)
 527                 sout, serr = locfg.run_noexcept(cmd)
 528
 529                 #p_ilo.wait()
 530                 #p_grep.wait()
 531                 if sout.strip() != "":
 532                         print "sout: %s" % sout.strip()
 533                         return sout.strip()
 534
 535                 if not dryrun:
 536                         locfg = soltesz.CMD()
 537                         cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
 538                                                 self.host, "iloxml/Reset_Server.xml",
 539                                                 self.username, self.password)
 540                         sout, serr = locfg.run_noexcept(cmd)
 541
 542                         #cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s'" % (
 543                         #               self.host, "iloxml/Reset_Server.xml",
 544                         #               self.username, self.password)
 545                         #print cmd
 546                         #p_ilo = Popen(cmd, stdin=PIPE, stdout=PIPE, shell=True)
 547                         #cmd2 = "grep 'MESSAGE' | grep -v 'No error'"
 548                         #p_grep = Popen(cmd2, shell=True, stdin=p_ilo.stdout, stdout=PIPE, stderr=PIPE)
 549                         #sout, serr = p_grep.communicate()
 550                         #try: p_ilo.wait()
 551                         #except:
 552                         #       print "p_ilo wait failed."
 553                         #       pass
 554                         #try: p_grep.wait()
 555                         #except:
 556                         #       print "p_grep wait failed."
 557                         #       pass
 558
 559                         if sout.strip() != "":
 560                                 print "sout: %s" % sout.strip()
 561                                 #return sout.strip()
 562                 return 0
 563
 564 class BayTechAU(PCUControl):
 565         def run(self, node_port, dryrun):
 566                 self.open(self.host, self.username, None, "Enter user name:")
 567                 self.sendPassword(self.password, "Enter Password:")
 568
 569                 #self.ifThenSend("RPC-16>", "Status")
 570                 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
 571
 572                 # Reboot Outlet  N        (Y/N)?
 573                 if dryrun:
 574                         self.ifThenSend("(Y/N)?", "N")
 575                 else:
 576                         self.ifThenSend("(Y/N)?", "Y")
 577                 self.ifThenSend("RPC3-NC>", "")
 578
 579                 self.close()
 580                 return 0
 581
 582 class BayTechGeorgeTown(PCUControl):
 583         def run(self, node_port, dryrun):
 584                 self.open(self.host, self.username, None, "Enter user name:")
 585                 self.sendPassword(self.password, "Enter Password:")
 586
 587                 #self.ifThenSend("RPC-16>", "Status")
 588
 589                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
 590
 591                 # Reboot Outlet  N        (Y/N)?
 592                 if dryrun:
 593                         self.ifThenSend("(Y/N)?", "N")
 594                 else:
 595                         self.ifThenSend("(Y/N)?", "Y")
 596                 self.ifThenSend("RPC-16>", "")
 597
 598                 self.close()
 599                 return 0
 600
 601 class BayTechCtrlCUnibe(PCUControl):
 602         """
 603                 For some reason, these units let you log in fine, but they hang
 604                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 605                 why.
 606         """
 607         def run(self, node_port, dryrun):
 608                 print "BayTechCtrlC %s" % self.host
 609
 610                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 611                 s = pxssh.pxssh()
 612                 if not s.login(self.host, self.username, self.password, ssh_options):
 613                         raise ExceptionPassword("Invalid Password")
 614                 # Otherwise, the login succeeded.
 615
 616                 # Send a ctrl-c to the remote process.
 617                 print "sending ctrl-c"
 618                 s.send(chr(3))
 619
 620                 # Control Outlets  (5 ,1).........5
 621                 try:
 622                         index = s.expect(["Enter Request :"])
 623
 624                         if index == 0:
 625                                 print "3"
 626                                 s.send("3\r\n")
 627                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 628                                 if index == 1:
 629                                         s.send(self.username + "\r\n")
 630                                         index = s.expect(["DS-RPC>"])
 631
 632                                 if index == 0:
 633                                         print "Reboot %d" % node_port
 634                                         s.send("Reboot %d\r\n" % node_port)
 635
 636                                         index = s.expect(["(Y/N)?"])
 637                                         if index == 0:
 638                                                 if dryrun:
 639                                                         print "sending N"
 640                                                         s.send("N\r\n")
 641                                                 else:
 642                                                         print "sending Y"
 643                                                         s.send("Y\r\n")
 644
 645                                 #index = s.expect(["DS-RPC>"])
 646                                 #print "got prompt back"
 647
 648                         s.close()
 649
 650                 except pexpect.EOF:
 651                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 652                 except pexpect.TIMEOUT:
 653                         raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
 654
 655                 return 0
 656
 657 class BayTechCtrlC(PCUControl):
 658         """
 659                 For some reason, these units let you log in fine, but they hang
 660                 indefinitely, unless you send a Ctrl-C after the password.  No idea
 661                 why.
 662         """
 663         def run(self, node_port, dryrun):
 664                 print "BayTechCtrlC %s" % self.host
 665
 666                 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
 667                 s = pxssh.pxssh()
 668                 if not s.login(self.host, self.username, self.password, ssh_options):
 669                         raise ExceptionPassword("Invalid Password")
 670                 # Otherwise, the login succeeded.
 671
 672                 # Send a ctrl-c to the remote process.
 673                 print "sending ctrl-c"
 674                 s.send(chr(3))
 675
 676                 # Control Outlets  (5 ,1).........5
 677                 try:
 678                         index = s.expect(["Enter Request :"])
 679
 680                         if index == 0:
 681                                 print "5"
 682                                 s.send("5\r\n")
 683                                 index = s.expect(["DS-RPC>", "Enter user name:"])
 684                                 if index == 1:
 685                                         print "sending username"
 686                                         s.send(self.username + "\r\n")
 687                                         index = s.expect(["DS-RPC>"])
 688
 689                                 if index == 0:
 690                                         print "Reboot %d" % node_port
 691                                         s.send("Reboot %d\r\n" % node_port)
 692
 693                                         index = s.expect(["(Y/N)?"])
 694                                         if index == 0:
 695                                                 if dryrun:
 696                                                         print "sending N"
 697                                                         s.send("N\r\n")
 698                                                 else:
 699                                                         print "sending Y"
 700                                                         s.send("Y\r\n")
 701
 702                                 index = s.expect(["DS-RPC>"])
 703                                 #print "got prompt back"
 704
 705                         s.close()
 706
 707                 except pexpect.EOF:
 708                         raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
 709                 except pexpect.TIMEOUT:
 710                         raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
 711
 712                 return 0
 713
 714 class BayTech(PCUControl):
 715         def run(self, node_port, dryrun):
 716                 self.open(self.host, self.username)
 717                 self.sendPassword(self.password)
 718
 719                 # Control Outlets  (5 ,1).........5
 720                 self.ifThenSend("Enter Request :", "5")
 721
 722                 # Reboot N
 723                 try:
 724                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
 725                 except ExceptionNotFound, msg:
 726                         # one machine is configured to ask for a username,
 727                         # even after login...
 728                         print "msg: %s" % msg
 729                         self.transport.write(self.username + "\r\n")
 730                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
 731
 732                 # Reboot Outlet  N        (Y/N)?
 733                 if dryrun:
 734                         self.ifThenSend("(Y/N)?", "N")
 735                 else:
 736                         self.ifThenSend("(Y/N)?", "Y")
 737                 self.ifThenSend("DS-RPC>", "")
 738
 739                 self.close()
 740                 return 0
 741
 742 class WTIIPS4(PCUControl):
 743         def run(self, node_port, dryrun):
 744                 self.open(self.host)
 745                 self.sendPassword(self.password, "Enter Password:")
 746
 747                 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
 748                 if not dryrun:
 749                         self.ifThenSend("Sure? (Y/N): ", "N")
 750                 else:
 751                         self.ifThenSend("Sure? (Y/N): ", "Y")
 752
 753                 self.ifThenSend("IPS> ", "")
 754
 755                 self.close()
 756                 return 0
 757
 758 class ePowerSwitchGood(PCUControl):
 759         # NOTE:
 760         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
 761         #               For some reason this both doesn't work and in some cases, actually
 762         #               hangs the PCU.  Definitely not what we want.
 763         #
 764         #               The code below is much simpler.  Just letting things fail first,
 765         #               and then, trying again with authentication string in the header.
 766         #
 767         def run(self, node_port, dryrun):
 768                 self.transport = None
 769                 self.url = "http://%s:%d/" % (self.host,80)
 770                 uri = "%s:%d" % (self.host,80)
 771
 772                 req = urllib2.Request(self.url)
 773                 try:
 774                         handle = urllib2.urlopen(req)
 775                 except IOError, e:
 776                         # NOTE: this is expected to fail initially
 777                         pass
 778                 else:
 779                         print self.url
 780                         print "-----------"
 781                         print handle.read()
 782                         print "-----------"
 783                         return "ERROR: not protected by HTTP authentication"
 784
 785                 if not hasattr(e, 'code') or e.code != 401:
 786                         return "ERROR: failed for: %s" % str(e)
 787
 788                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
 789                 # NOTE: assuming basic realm authentication.
 790                 authheader = "Basic %s" % base64data
 791                 req.add_header("Authorization", authheader)
 792
 793                 try:
 794                         f = urllib2.urlopen(req)
 795                 except IOError, e:
 796                         # failing here means the User/passwd is wrong (hopefully)
 797                         raise ExceptionPassword("Incorrect username/password")
 798
 799                 # TODO: after verifying that the user/password is correct, we should
 800                 # actually reboot the given node.
 801
 802                 if not dryrun:
 803                         # add data to handler,
 804                         # fetch url one more time on cmd.html, econtrol.html or whatever.
 805                         pass
 806
 807                 if self.verbose: print f.read()
 808
 809                 self.close()
 810                 return 0
 811
 812
 813 class ePowerSwitchOld(PCUControl):
 814         def run(self, node_port, dryrun):
 815                 self.url = "http://%s:%d/" % (self.host,80)
 816                 uri = "%s:%d" % (self.host,80)
 817
 818                 # create authinfo
 819                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 820                 authinfo.add_password (None, uri, self.username, self.password)
 821                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 822
 823                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 824                 transport = urllib2.build_opener(authinfo)
 825                 f = transport.open(self.url)
 826                 if self.verbose: print f.read()
 827
 828                 if not dryrun:
 829                         transport = urllib2.build_opener(authhandler)
 830                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
 831                         if self.verbose: print f.read()
 832
 833                 self.close()
 834                 return 0
 835
 836 class ePowerSwitch(PCUControl):
 837         def run(self, node_port, dryrun):
 838                 self.url = "http://%s:%d/" % (self.host,80)
 839                 uri = "%s:%d" % (self.host,80)
 840
 841                 # TODO: I'm still not sure what the deal is here.
 842                 #               two independent calls appear to need to be made before the
 843                 #               reboot will succeed.  It doesn't seem to be possible to do
 844                 #               this with a single call.  I have no idea why.
 845
 846                 # create authinfo
 847                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 848                 authinfo.add_password (None, uri, self.username, self.password)
 849                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 850
 851                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
 852                 transport = urllib2.build_opener()
 853                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
 854                 if self.verbose: print f.read()
 855
 856                 if not dryrun:
 857                         transport = urllib2.build_opener(authhandler)
 858                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
 859                         if self.verbose: print f.read()
 860
 861                 #       data= "P%d=r" % node_port
 862                 #self.open(self.host, self.username, self.password)
 863                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
 864                 #self.sendHTTP("econtrol.html", data)
 865                 #self.sendHTTP("cmd.html", data)
 866
 867                 self.close()
 868                 return 0
 869
 870
 871 ### rebooting european BlackBox PSE boxes
 872 # Thierry Parmentelat - May 11 2005
 873 # tested on 4-ports models known as PSE505-FR
 874 # uses http to POST a data 'P<port>=r'
 875 # relies on basic authentication within http1.0
 876 # first curl-based script was
 877 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
 878 #       http://<hostname>:<http_port>/cmd.html && echo OK
 879
 880 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
 881
 882         global verbose
 883
 884         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
 885         data= "P%d=r" % port_in_pcu
 886         if verbose:
 887                 logger.debug("POSTing '%s' on %s" % (data,url))
 888
 889         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
 890         uri = "%s:%d" % (pcu_ip,http_port)
 891         authinfo.add_password (None, uri, username, password)
 892         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
 893
 894         opener = urllib2.build_opener(authhandler)
 895         urllib2.install_opener(opener)
 896
 897         if (dryrun):
 898                 return 0
 899
 900         try:
 901                 f = urllib2.urlopen(url,data)
 902
 903                 r= f.read()
 904                 if verbose:
 905                         logger.debug(r)
 906                 return 0
 907
 908         except urllib2.URLError,err:
 909                 logger.info('Could not open http connection', err)
 910                 return "bbpse error"
 911
 912 ### rebooting x10toggle based systems addressed by port
 913 # Marc E. Fiuczynski - May 31 2005
 914 # tested on 4-ports models known as PSE505-FR
 915 # uses ssh and password to login to an account
 916 # that will cause the system to be powercycled.
 917
 918 def x10toggle_reboot(ip, username, password, port, dryrun):
 919         global verbose
 920
 921         ssh = None
 922         try:
 923                 ssh = pyssh.Ssh(username, ip)
 924                 ssh.open()
 925
 926                 # Login
 927                 telnet_answer(ssh, "password:", password)
 928
 929                 if not dryrun:
 930                         # Reboot
 931                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
 932
 933                 # Close
 934                 output = ssh.close()
 935                 if verbose:
 936                         logger.debug(output)
 937                 return 0
 938
 939         except Exception, err:
 940                 if verbose:
 941                         logger.debug(err)
 942                 if ssh:
 943                         output = ssh.close()
 944                         if verbose:
 945                                 logger.debug(output)
 946                 return errno.ETIMEDOUT
 947
 948 ### rebooting Dell systems via RAC card
 949 # Marc E. Fiuczynski - June 01 2005
 950 # tested with David Lowenthal's itchy/scratchy nodes at UGA
 951 #
 952
 953 def runcmd(command, args, username, password, timeout = None):
 954
 955         result = [None]
 956         result_ready = threading.Condition()
 957
 958         def set_result(x):
 959
 960                 result_ready.acquire()
 961                 try:
 962                         result[0] = x
 963                 finally:
 964                         result_ready.notify()
 965                         result_ready.release()
 966
 967         def do_command(command, username, password):
 968
 969                 try:
 970                         # Popen4 is a popen-type class that combines stdout and stderr
 971                         p = popen2.Popen4(command)
 972
 973                         # read all output data
 974                         p.tochild.write("%s\n" % username)
 975                         p.tochild.write("%s\n" % password)
 976                         p.tochild.close()
 977                         data = p.fromchild.read()
 978
 979                         while True:
 980                                 # might get interrupted by a signal in poll() or waitpid()
 981                                 try:
 982                                         retval = p.wait()
 983                                         set_result((retval, data))
 984                                         break
 985                                 except OSError, ex:
 986                                         if ex.errno == errno.EINTR:
 987                                                 continue
 988                                         raise ex
 989                 except Exception, ex:
 990                         set_result(ex)
 991
 992         if args:
 993                 command = " ".join([command] + args)
 994
 995         worker = threading.Thread(target = do_command, args = (command, username, password, ))
 996         worker.setDaemon(True)
 997         result_ready.acquire()
 998         worker.start()
 999         result_ready.wait(timeout)
1000         try:
1001                 if result == [None]:
1002                         raise Exception, "command timed-out: '%s'" % command
1003         finally:
1004                 result_ready.release()
1005         result = result[0]
1006
1007         if isinstance(result, Exception):
1008                 raise result
1009         else:
1010                 (retval, data) = result
1011                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1012                         return data
1013                 else:
1014                         out = "system command ('%s') " % command
1015                         if os.WIFEXITED(retval):
1016                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1017                         else:
1018                                 out += "killed by signal %d" % os.WTERMSIG(retval)
1019                         if data:
1020                                 out += "; output follows:\n" + data
1021                         raise Exception, out
1022
1023 def racadm_reboot(host, username, password, port, dryrun):
1024         global verbose
1025
1026         ip = socket.gethostbyname(host)
1027         try:
1028                 cmd = "/usr/sbin/racadm"
1029                 os.stat(cmd)
1030                 if not dryrun:
1031                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1032                                 username, password)
1033                 else:
1034                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1035                                 username, password)
1036
1037                 print "RUNCMD: %s" % output
1038                 if verbose:
1039                         logger.debug(output)
1040                 return 0
1041
1042         except Exception, err:
1043                 logger.debug("runcmd raised exception %s" % err)
1044                 if verbose:
1045                         logger.debug(err)
1046                 return -1
1047
1048 def pcu_name(pcu):
1049         if pcu['hostname'] is not None and pcu['hostname'] is not "":
1050                 return pcu['hostname']
1051         elif pcu['ip'] is not None and pcu['ip'] is not "":
1052                 return pcu['ip']
1053         else:
1054                 return None
1055
1056 def get_pcu_values(pcu_id):
1057         # TODO: obviously, this shouldn't be loaded each time...
1058         import soltesz
1059         fb =soltesz.dbLoad("findbadpcus")
1060
1061         try:
1062                 values = fb['nodes']["id_%s" % pcu_id]['values']
1063         except:
1064                 values = None
1065
1066         return values
1067
1068 def reboot(nodename):
1069         return reboot_policy(nodename, True, False)
1070
1071 def reboot_policy(nodename, continue_probe, dryrun):
1072         global verbose
1073
1074         pcu = plc.getpcu(nodename)
1075         if not pcu:
1076                 return False # "%s has no pcu" % nodename
1077
1078         values = get_pcu_values(pcu['pcu_id'])
1079         if values == None:
1080                 return False #"no info for pcu_id %s" % pcu['pcu_id']
1081
1082         # Try the PCU first
1083         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1084
1085         ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1086
1087         if ret != 0:
1088                 print ret
1089                 return False
1090         else:
1091                 return True
1092
1093 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1094         rb_ret = ""
1095
1096         try:
1097                 # DataProbe iPal (many sites)
1098                 if  continue_probe and values['model'].find("Dataprobe IP-41x/IP-81x") >= 0:
1099                         ipal = IPAL(values, verbose, ['23'])
1100                         rb_ret = ipal.reboot(values[nodename], dryrun)
1101
1102                 # APC Masterswitch (Berkeley)
1103                 elif continue_probe and values['model'].find("APC AP79xx/Masterswitch") >= 0:
1104
1105                         # TODO: make a more robust version of APC
1106                         if values['pcu_id'] in [1163,1055,1111,1231,1113,1127,1128,1148]:
1107                                 apc = APCEurope(values, verbose, ['22', '23'])
1108                                 rb_ret = apc.reboot(values[nodename], dryrun)
1109
1110                         elif values['pcu_id'] in [1110,86]:
1111                                 apc = APCBrazil(values, verbose, ['22', '23'])
1112                                 rb_ret = apc.reboot(values[nodename], dryrun)
1113
1114                         elif values['pcu_id'] in [1221,1225]:
1115                                 apc = APCBerlin(values, verbose, ['22', '23'])
1116                                 rb_ret = apc.reboot(values[nodename], dryrun)
1117
1118                         elif values['pcu_id'] in [1173,1221,1220]:
1119                                 apc = APCFolsom(values, verbose, ['22', '23'])
1120                                 rb_ret = apc.reboot(values[nodename], dryrun)
1121
1122                         else:
1123                                 apc = APCMaster(values, verbose, ['22', '23'])
1124                                 rb_ret = apc.reboot(values[nodename], dryrun)
1125
1126                 # BayTech DS4-RPC
1127                 elif continue_probe and values['model'].find("Baytech DS4-RPC") >= 0:
1128                         if values['pcu_id'] in [1237,1052,1209,1002,1008,1041,1013,1022]:
1129                                 # These  require a 'ctrl-c' to be sent...
1130                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1131                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1132
1133                         elif values['pcu_id'] in [93]:
1134                                 baytech = BayTechAU(values, verbose, ['22', '23'])
1135                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1136
1137                         elif values['pcu_id'] in [1057]:
1138                                 # These  require a 'ctrl-c' to be sent...
1139                                 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1140                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1141
1142                         elif values['pcu_id'] in [1012]:
1143                                 # This pcu sometimes doesn't present the 'Username' prompt,
1144                                 # unless you immediately try again...
1145                                 try:
1146                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1147                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1148                                 except:
1149                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1150                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1151                         else:
1152                                 baytech = BayTech(values, verbose, ['22', '23'])
1153                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1154
1155                 # iLO
1156                 elif continue_probe and values['model'].find("HP iLO") >= 0:
1157                         try:
1158                                 hpilo = HPiLO(values, verbose, ['22'])
1159                                 rb_ret = hpilo.reboot(0, dryrun)
1160                                 if rb_ret != 0:
1161                                         hpilo = HPiLOHttps(values, verbose, ['443'])
1162                                         rb_ret = hpilo.reboot(0, dryrun)
1163                         except:
1164                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1165                                 rb_ret = hpilo.reboot(0, dryrun)
1166
1167                 # DRAC ssh
1168                 elif continue_probe and values['model'].find("Dell RAC") >= 0:
1169                         # TODO: I don't think DRACRacAdm will throw an exception for the
1170                         # default method to catch...
1171                         try:
1172                                 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1173                                 rb_ret = drac.reboot(0, dryrun)
1174                         except:
1175                                 drac = DRAC(values, verbose, ['22'])
1176                                 rb_ret = drac.reboot(0, dryrun)
1177
1178                 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1179                                 wti = WTIIPS4(values, verbose, ['23'])
1180                                 rb_ret = wti.reboot(values[nodename], dryrun)
1181
1182                 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1183                 elif continue_probe and \
1184                         (values['model'].find("BlackBox PS5xx") >= 0 or
1185                          values['model'].find("ePowerSwitch 1/4/8x") >=0 ):
1186
1187                         # TODO: allow a different port than http 80.
1188                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1189                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1190                         elif values['pcu_id'] in [1003]:
1191                                 eps = ePowerSwitch(values, verbose, ['80'])
1192                         else:
1193                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1194
1195                         rb_ret = eps.reboot(values[nodename], dryrun)
1196
1197                 elif continue_probe:
1198                         rb_ret = "Unsupported_PCU"
1199
1200                 elif continue_probe == False:
1201                         if 'portstatus' in values:
1202                                 rb_ret = "NetDown"
1203                         else:
1204                                 rb_ret = "Not_Run"
1205                 else:
1206                         rb_ret = -1
1207
1208         except ExceptionPort, err:
1209                 rb_ret = str(err)
1210
1211         return rb_ret
1212         # ????
1213         #elif continue_probe and values['protocol'] == "racadm" and \
1214         #               values['model'] == "RAC":
1215         #       rb_ret = racadm_reboot(pcu_name(values),
1216         #                                                                 values['username'],
1217         #                                                                 values['password'],
1218         #                                                                 pcu[nodename],
1219         #                                                                 dryrun)
1220
1221 def main():
1222         logger.setLevel(logging.DEBUG)
1223         ch = logging.StreamHandler()
1224         ch.setLevel(logging.DEBUG)
1225         formatter = logging.Formatter('LOGGER - %(message)s')
1226         ch.setFormatter(formatter)
1227         logger.addHandler(ch)
1228
1229         try:
1230                 for node in sys.argv[1:]:
1231                         print "Rebooting %s" % node
1232                         if reboot_policy(node, True, False):
1233                                 print "success"
1234                         else:
1235                                 print "failed"
1236         except Exception, err:
1237                 print err
1238
1239 if __name__ == '__main__':
1240         import plc
1241         logger = logging.getLogger("monitor")
1242         main()