3 # Reboot specified nodes
9 import errno, time, traceback
12 import threading, popen2
18 from subprocess import PIPE, Popen
19 import ssh.pxssh as pxssh
20 import ssh.pexpect as pexpect
23 # Use our versions of telnetlib and pyssh
24 sys.path.insert(0, os.path.dirname(sys.argv[0]))
26 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
32 # Event class ID from pcu events
33 #NODE_POWER_CONTROL = 3
36 #MONITOR_USER_ID = 11142
39 logger = logging.getLogger("monitor")
43 class ExceptionNoTransport(Exception): pass
44 class ExceptionNotFound(Exception): pass
45 class ExceptionPassword(Exception): pass
46 class ExceptionTimeout(Exception): pass
47 class ExceptionPrompt(Exception): pass
48 class ExceptionSequence(Exception): pass
49 class ExceptionReset(Exception): pass
50 class ExceptionPort(Exception): pass
51 class ExceptionUsername(Exception): pass
53 def telnet_answer(telnet, expected, buffer):
56 output = telnet.read_until(expected, TELNET_TIMEOUT)
58 # logger.debug(output)
59 if output.find(expected) == -1:
60 raise ExceptionNotFound, "'%s' not found" % expected
62 telnet.write(buffer + "\r\n")
65 # PCU has model, host, preferred-port, user, passwd,
67 # This is an object derived directly form the PLCAPI DB fields
69 def __init__(self, plc_pcu_dict):
70 for field in ['username', 'password', 'site_id',
73 'node_ids', 'ports', ]:
74 if field in plc_pcu_dict:
75 self.__setattr__(field, plc_pcu_dict[field])
77 raise Exception("No such field %s in PCU object" % field)
79 # These are the convenience functions build around the PCU object.
81 def __init__(self, plc_pcu_dict):
82 PCU.__init__(self, plc_pcu_dict)
83 self.host = self.pcu_name()
86 if self.hostname is not None and self.hostname is not "":
88 elif self.ip is not None and self.ip is not "":
93 def nodeidToPort(self, node_id):
94 if node_id in self.node_ids:
95 for i in range(0, len(self.node_ids)):
96 if node_id == self.node_ids[i]:
99 raise Exception("No such Node ID: %d" % node_id)
101 # This class captures the observed pcu records from FindBadPCUs.py
103 def __init__(self, pcu_record_dict):
104 for field in ['nodenames', 'portstatus',
107 if field in pcu_record_dict:
108 if field == "reboot":
109 self.__setattr__("reboot_str", pcu_record_dict[field])
111 self.__setattr__(field, pcu_record_dict[field])
113 raise Exception("No such field %s in pcu record dict" % field)
123 def __init__(self, type, verbose):
125 self.verbose = verbose
126 self.transport = None
128 def open(self, host, username=None, password=None, prompt="User Name"):
131 if self.type == self.TELNET:
132 transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
133 transport.set_debuglevel(self.verbose)
134 if username is not None:
135 self.transport = transport
136 self.ifThenSend(prompt, username, ExceptionUsername)
138 elif self.type == self.SSH:
139 if username is not None:
140 transport = pyssh.Ssh(username, host)
141 transport.set_debuglevel(self.verbose)
143 # TODO: have an ssh set_debuglevel() also...
145 raise Exception("Username cannot be None for ssh transport.")
146 elif self.type == self.HTTP:
147 self.url = "http://%s:%d/" % (host,80)
148 uri = "%s:%d" % (host,80)
151 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
152 authinfo.add_password (None, uri, username, password)
153 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
155 transport = urllib2.build_opener(authhandler)
158 raise Exception("Unknown transport type: %s" % self.type)
160 self.transport = transport
164 if self.type == self.TELNET:
165 self.transport.close()
166 elif self.type == self.SSH:
167 self.transport.close()
168 elif self.type == self.HTTP:
171 raise Exception("Unknown transport type %s" % self.type)
172 self.transport = None
174 def sendHTTP(self, resource, data):
176 print "POSTing '%s' to %s" % (data,self.url + resource)
179 f = self.transport.open(self.url + resource ,data)
184 except urllib2.URLError,err:
185 logger.info('Could not open http connection', err)
186 return "http transport error"
190 def sendPassword(self, password, prompt=None):
191 if self.type == self.TELNET:
193 self.ifThenSend("Password", password, ExceptionPassword)
195 self.ifThenSend(prompt, password, ExceptionPassword)
196 elif self.type == self.SSH:
197 self.ifThenSend("password:", password, ExceptionPassword)
198 elif self.type == self.HTTP:
201 raise Exception("Unknown transport type: %s" % self.type)
203 def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
205 if self.transport != None:
206 output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
207 if output.find(expected) == -1:
208 raise ErrorClass, "'%s' not found" % expected
210 self.transport.write(buffer + "\r\n")
212 raise ExceptionNoTransport("transport object is type None")
214 def ifElse(self, expected, ErrorClass):
216 self.transport.read_until(expected, self.TELNET_TIMEOUT)
218 raise ErrorClass("Could not find '%s' within timeout" % expected)
221 class PCUControl(Transport,PCUModel,PCURecord):
222 def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
223 PCUModel.__init__(self, plc_pcu_record)
224 PCURecord.__init__(self, plc_pcu_record)
227 if '22' in supported_ports and self.portstatus['22'] == "open":
229 elif '23' in supported_ports and self.portstatus['23'] == "open":
230 type = Transport.TELNET
231 elif '80' in supported_ports and self.portstatus['80'] == "open":
232 type = Transport.HTTP
233 elif '443' in supported_ports and self.portstatus['443'] == "open":
234 type = Transport.HTTP
235 elif '5869' in supported_ports and self.portstatus['5869'] == "open":
236 # For DRAC cards. Racadm opens this port.
237 type = Transport.HTTP
238 elif '9100' in supported_ports and self.portstatus['9100'] == "open":
239 type = Transport.IPAL
240 elif '16992' in supported_ports and self.portstatus['16992'] == "open":
241 type = Transport.HTTP
243 raise ExceptionPort("Unsupported Port: No transport from open ports")
245 raise Exception("No Portstatus: No transport because no open ports")
246 Transport.__init__(self, type, verbose)
248 def run(self, node_port, dryrun):
249 """ This function is to be defined by the specific PCU instance. """
252 def reboot(self, node_port, dryrun):
254 return self.run(node_port, dryrun)
255 except ExceptionNotFound, err:
256 return "error: " + str(err)
257 except ExceptionPassword, err:
258 return "password exception: " + str(err)
259 except ExceptionTimeout, err:
260 return "timeout exception: " + str(err)
261 except ExceptionUsername, err:
262 return "exception: no username prompt: " + str(err)
263 except ExceptionSequence, err:
264 return "sequence error: " + str(err)
265 except ExceptionPrompt, err:
266 return "prompt exception: " + str(err)
267 except ExceptionPort, err:
268 return "no ports exception: " + str(err)
269 except socket.error, err:
270 return "socket error: timeout: " + str(err)
271 except EOFError, err:
273 logger.debug("reboot: EOF")
275 self.transport.close()
277 traceback.print_exc()
278 return "EOF connection reset" + str(err)
280 class IPAL(PCUControl):
282 This now uses a proprietary format for communicating with the PCU. I
283 prefer it to Telnet, and Web access, since it's much lighter weight
284 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
287 def format_msg(self, data, cmd):
288 esc = chr(int('1b',16))
289 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
291 def recv_noblock(self, s, count):
295 # TODO: make sleep backoff, before stopping.
297 ret = s.recv(count, socket.MSG_DONTWAIT)
298 except socket.error, e:
299 if e[0] == errno.EAGAIN:
300 return Exception(e[1])
302 # TODO: not other exceptions.
306 def run(self, node_port, dryrun):
312 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
315 s.connect((self.host, 9100))
316 except socket.error, e:
318 if e[0] == errno.ECONNREFUSED:
319 # cannot connect to remote host
320 return Exception(e[1])
322 # TODO: what other conditions are there?
326 print "Checking status"
327 s.send(self.format_msg("", 'O'))
328 ret = self.recv_noblock(s, 8)
329 print "Current status is '%s'" % ret
331 if node_port < len(ret):
332 status = ret[node_port]
340 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
342 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
346 print "Pulsing %s" % node_port
348 s.send(self.format_msg("%s" % node_port, 'P'))
350 # NOTE: turn power on before pulsing the port.
351 print "power was off, so turning on then pulsing..."
352 s.send(self.format_msg("%s" % node_port, 'E'))
353 s.send(self.format_msg("%s" % node_port, 'P'))
355 print "Receiving response."
356 ret = self.recv_noblock(s, 8)
357 print "Current status is '%s'" % ret
359 if node_port < len(ret):
360 status = ret[node_port]
368 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
370 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
375 return "Failed Power On"
380 # TELNET version of protocol...
381 # #self.open(self.host)
382 # ## XXX Some iPals require you to hit Enter a few times first
383 # #self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
385 # self.ifThenSend("Password >", self.password, ExceptionPassword)
386 # self.transport.write("\r\n\r\n")
387 # if not dryrun: # P# - Pulse relay
388 # print "node_port %s" % node_port
389 # self.ifThenSend("Enter >",
390 # "P7", # % node_port,
392 # print "send newlines"
393 # self.transport.write("\r\n\r\n")
394 # print "after new lines"
395 # # Get the next prompt
396 # print "wait for enter"
397 # self.ifElse("Enter >", ExceptionTimeout)
402 class APCEurope(PCUControl):
403 def run(self, node_port, dryrun):
404 self.open(self.host, self.username)
405 self.sendPassword(self.password)
407 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
408 self.ifThenSend("\r\n> ", "2")
409 self.ifThenSend("\r\n> ", str(node_port))
410 # 3- Immediate Reboot
411 self.ifThenSend("\r\n> ", "3")
414 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
418 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
419 "", ExceptionSequence)
420 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
425 class APCBrazil(PCUControl):
426 def run(self, node_port, dryrun):
427 self.open(self.host, self.username)
428 self.sendPassword(self.password)
430 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
431 self.ifThenSend("\r\n> ", str(node_port))
432 # 4- Immediate Reboot
433 self.ifThenSend("\r\n> ", "4")
436 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
440 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
441 "", ExceptionSequence)
442 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
447 class APCBerlin(PCUControl):
448 def run(self, node_port, dryrun):
449 self.open(self.host, self.username)
450 self.sendPassword(self.password)
452 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
453 self.ifThenSend("\r\n> ", "2")
454 self.ifThenSend("\r\n> ", "1")
455 self.ifThenSend("\r\n> ", str(node_port))
456 # 3- Immediate Reboot
457 self.ifThenSend("\r\n> ", "3")
460 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
464 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
465 "", ExceptionSequence)
466 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
471 class APCFolsom(PCUControl):
472 def run(self, node_port, dryrun):
473 self.open(self.host, self.username)
474 self.sendPassword(self.password)
476 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
477 self.ifThenSend("\r\n> ", "2")
478 self.ifThenSend("\r\n> ", "1")
479 self.ifThenSend("\r\n> ", str(node_port))
480 self.ifThenSend("\r\n> ", "1")
482 # 3- Immediate Reboot
483 self.ifThenSend("\r\n> ", "3")
486 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
490 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
491 "", ExceptionSequence)
492 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
497 class APCMaster(PCUControl):
498 def run(self, node_port, dryrun):
499 print "Rebooting %s" % self.host
500 self.open(self.host, self.username)
501 self.sendPassword(self.password)
504 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
505 # 3- Outlet Control/Config
506 self.ifThenSend("\r\n> ", "3")
508 self.ifThenSend("\r\n> ", str(node_port))
510 self.ifThenSend("\r\n> ", "1")
511 # 3- Immediate Reboot
512 self.ifThenSend("\r\n> ", "3")
515 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
519 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
520 "", ExceptionSequence)
521 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
526 class APC(PCUControl):
527 def __init__(self, plc_pcu_record, verbose):
528 PCUControl.__init__(self, plc_pcu_record, verbose)
530 self.master = APCMaster(plc_pcu_record, verbose)
531 self.folsom = APCFolsom(plc_pcu_record, verbose)
532 self.europe = APCEurope(plc_pcu_record, verbose)
534 def run(self, node_port, dryrun):
538 for pcu in [self.master, self.europe, self.folsom]:
541 print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
544 time.sleep(sleep_time)
545 ret = pcu.reboot(node_port, dryrun)
546 except ExceptionSequence, err:
552 return "Unknown reboot sequence for APC PCU"
556 class IntelAMT(PCUControl):
557 def run(self, node_port, dryrun):
561 cmd_str = "IntelAMTSDK/Samples/RemoteControl/remoteControl"
564 # NOTE: -p checks the power state of the host.
565 # TODO: parse the output to find out if it's ok or not.
566 cmd_str += " -p http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
568 cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
571 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
573 class DRACRacAdm(PCUControl):
574 def run(self, node_port, dryrun):
576 print "trying racadm_reboot..."
577 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
581 class DRAC(PCUControl):
582 def run(self, node_port, dryrun):
583 self.open(self.host, self.username)
584 self.sendPassword(self.password)
586 print "logging in..."
587 self.transport.write("\r\n")
590 self.ifThenSend("[%s]#" % self.username, "getsysinfo")
593 self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
595 self.ifThenSend("[%s]#" % self.username, "exit")
600 class HPiLO(PCUControl):
601 def run(self, node_port, dryrun):
602 self.open(self.host, self.username)
603 self.sendPassword(self.password)
606 self.ifThenSend("</>hpiLO->", "cd system1")
608 # Reboot Outlet N (Y/N)?
610 self.ifThenSend("</system1>hpiLO->", "POWER")
613 self.ifThenSend("</system1>hpiLO->", "reset")
615 self.ifThenSend("</system1>hpiLO->", "exit")
621 class HPiLOHttps(PCUControl):
622 def run(self, node_port, dryrun):
625 locfg = soltesz.CMD()
626 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
627 self.host, "iloxml/Get_Network.xml",
628 self.username, self.password)
629 sout, serr = locfg.run_noexcept(cmd)
631 if sout.strip() != "":
632 print "sout: %s" % sout.strip()
636 locfg = soltesz.CMD()
637 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
638 self.host, "iloxml/Reset_Server.xml",
639 self.username, self.password)
640 sout, serr = locfg.run_noexcept(cmd)
642 if sout.strip() != "":
643 print "sout: %s" % sout.strip()
647 class BayTechAU(PCUControl):
648 def run(self, node_port, dryrun):
649 self.open(self.host, self.username, None, "Enter user name:")
650 self.sendPassword(self.password, "Enter Password:")
652 #self.ifThenSend("RPC-16>", "Status")
653 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
655 # Reboot Outlet N (Y/N)?
657 self.ifThenSend("(Y/N)?", "N")
659 self.ifThenSend("(Y/N)?", "Y")
660 self.ifThenSend("RPC3-NC>", "")
665 class BayTechGeorgeTown(PCUControl):
666 def run(self, node_port, dryrun):
667 self.open(self.host, self.username, None, "Enter user name:")
668 self.sendPassword(self.password, "Enter Password:")
670 #self.ifThenSend("RPC-16>", "Status")
672 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
674 # Reboot Outlet N (Y/N)?
676 self.ifThenSend("(Y/N)?", "N")
678 self.ifThenSend("(Y/N)?", "Y")
679 self.ifThenSend("RPC-16>", "")
684 class BayTechCtrlCUnibe(PCUControl):
686 For some reason, these units let you log in fine, but they hang
687 indefinitely, unless you send a Ctrl-C after the password. No idea
690 def run(self, node_port, dryrun):
691 print "BayTechCtrlC %s" % self.host
693 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
695 if not s.login(self.host, self.username, self.password, ssh_options):
696 raise ExceptionPassword("Invalid Password")
697 # Otherwise, the login succeeded.
699 # Send a ctrl-c to the remote process.
700 print "sending ctrl-c"
703 # Control Outlets (5 ,1).........5
705 index = s.expect(["Enter Request :"])
710 index = s.expect(["DS-RPC>", "Enter user name:"])
712 s.send(self.username + "\r\n")
713 index = s.expect(["DS-RPC>"])
716 print "Reboot %d" % node_port
717 s.send("Reboot %d\r\n" % node_port)
719 index = s.expect(["(Y/N)?"])
728 #index = s.expect(["DS-RPC>"])
729 #print "got prompt back"
734 raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
735 except pexpect.TIMEOUT:
736 raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
740 class BayTechCtrlC(PCUControl):
742 For some reason, these units let you log in fine, but they hang
743 indefinitely, unless you send a Ctrl-C after the password. No idea
746 def run(self, node_port, dryrun):
747 print "BayTechCtrlC %s" % self.host
749 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
751 if not s.login(self.host, self.username, self.password, ssh_options):
752 raise ExceptionPassword("Invalid Password")
753 # Otherwise, the login succeeded.
755 # Send a ctrl-c to the remote process.
756 print "sending ctrl-c"
759 # Control Outlets (5 ,1).........5
761 index = s.expect(["Enter Request :"])
766 index = s.expect(["DS-RPC>", "Enter user name:"])
768 print "sending username"
769 s.send(self.username + "\r\n")
770 index = s.expect(["DS-RPC>"])
773 print "Reboot %d" % node_port
774 s.send("Reboot %d\r\n" % node_port)
776 index = s.expect(["(Y/N)?"])
785 index = s.expect(["DS-RPC>"])
786 #print "got prompt back"
791 raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
792 except pexpect.TIMEOUT:
793 raise ExceptionPrompt("Timeout before Prompt")
797 class BayTech(PCUControl):
798 def run(self, node_port, dryrun):
799 self.open(self.host, self.username)
800 self.sendPassword(self.password)
802 # Control Outlets (5 ,1).........5
803 self.ifThenSend("Enter Request :", "5")
807 self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
808 except ExceptionNotFound, msg:
809 # one machine is configured to ask for a username,
810 # even after login...
811 print "msg: %s" % msg
812 self.transport.write(self.username + "\r\n")
813 self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
815 # Reboot Outlet N (Y/N)?
817 self.ifThenSend("(Y/N)?", "N")
819 self.ifThenSend("(Y/N)?", "Y")
820 self.ifThenSend("DS-RPC>", "")
825 class WTIIPS4(PCUControl):
826 def run(self, node_port, dryrun):
828 self.sendPassword(self.password, "Enter Password:")
830 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
832 self.ifThenSend("Sure? (Y/N): ", "N")
834 self.ifThenSend("Sure? (Y/N): ", "Y")
836 self.ifThenSend("IPS> ", "")
841 class ePowerSwitchGood(PCUControl):
843 # The old code used Python's HTTPPasswordMgrWithDefaultRealm()
844 # For some reason this both doesn't work and in some cases, actually
845 # hangs the PCU. Definitely not what we want.
847 # The code below is much simpler. Just letting things fail first,
848 # and then, trying again with authentication string in the header.
850 def run(self, node_port, dryrun):
851 self.transport = None
852 self.url = "http://%s:%d/" % (self.host,80)
853 uri = "%s:%d" % (self.host,80)
855 req = urllib2.Request(self.url)
857 handle = urllib2.urlopen(req)
859 # NOTE: this is expected to fail initially
866 return "ERROR: not protected by HTTP authentication"
868 if not hasattr(e, 'code') or e.code != 401:
869 return "ERROR: failed for: %s" % str(e)
871 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
872 # NOTE: assuming basic realm authentication.
873 authheader = "Basic %s" % base64data
874 req.add_header("Authorization", authheader)
877 f = urllib2.urlopen(req)
879 # failing here means the User/passwd is wrong (hopefully)
880 raise ExceptionPassword("Incorrect username/password")
882 # NOTE: after verifying that the user/password is correct,
883 # actually reboot the given node.
886 data = urllib.urlencode({'P%d' % node_port : "r"})
887 req = urllib2.Request(self.url + "cmd.html")
888 req.add_header("Authorization", authheader)
889 # add data to handler,
890 f = urllib2.urlopen(req, data)
891 if self.verbose: print f.read()
893 import traceback; traceback.print_exc()
895 # fetch url one more time on cmd.html, econtrol.html or whatever.
898 if self.verbose: print f.read()
904 class ePowerSwitchOld(PCUControl):
905 def run(self, node_port, dryrun):
906 self.url = "http://%s:%d/" % (self.host,80)
907 uri = "%s:%d" % (self.host,80)
910 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
911 authinfo.add_password (None, uri, self.username, self.password)
912 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
914 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
915 transport = urllib2.build_opener(authinfo)
916 f = transport.open(self.url)
917 if self.verbose: print f.read()
920 transport = urllib2.build_opener(authhandler)
921 f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
922 if self.verbose: print f.read()
927 class ePowerSwitch(PCUControl):
928 def run(self, node_port, dryrun):
929 self.url = "http://%s:%d/" % (self.host,80)
930 uri = "%s:%d" % (self.host,80)
932 # TODO: I'm still not sure what the deal is here.
933 # two independent calls appear to need to be made before the
934 # reboot will succeed. It doesn't seem to be possible to do
935 # this with a single call. I have no idea why.
938 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
939 authinfo.add_password (None, uri, self.username, self.password)
940 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
942 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
943 transport = urllib2.build_opener()
944 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
945 if self.verbose: print f.read()
948 transport = urllib2.build_opener(authhandler)
949 f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
950 if self.verbose: print f.read()
952 # data= "P%d=r" % node_port
953 #self.open(self.host, self.username, self.password)
954 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
955 #self.sendHTTP("econtrol.html", data)
956 #self.sendHTTP("cmd.html", data)
962 ### rebooting european BlackBox PSE boxes
963 # Thierry Parmentelat - May 11 2005
964 # tested on 4-ports models known as PSE505-FR
965 # uses http to POST a data 'P<port>=r'
966 # relies on basic authentication within http1.0
967 # first curl-based script was
968 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
969 # http://<hostname>:<http_port>/cmd.html && echo OK
971 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
975 url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
976 data= "P%d=r" % port_in_pcu
978 logger.debug("POSTing '%s' on %s" % (data,url))
980 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
981 uri = "%s:%d" % (pcu_ip,http_port)
982 authinfo.add_password (None, uri, username, password)
983 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
985 opener = urllib2.build_opener(authhandler)
986 urllib2.install_opener(opener)
992 f = urllib2.urlopen(url,data)
999 except urllib2.URLError,err:
1000 logger.info('Could not open http connection', err)
1001 return "bbpse error"
1003 ### rebooting x10toggle based systems addressed by port
1004 # Marc E. Fiuczynski - May 31 2005
1005 # tested on 4-ports models known as PSE505-FR
1006 # uses ssh and password to login to an account
1007 # that will cause the system to be powercycled.
1009 def x10toggle_reboot(ip, username, password, port, dryrun):
1014 ssh = pyssh.Ssh(username, ip)
1018 telnet_answer(ssh, "password:", password)
1022 telnet_answer(ssh, "x10toggle>", "A%d" % port)
1025 output = ssh.close()
1027 logger.debug(output)
1030 except Exception, err:
1034 output = ssh.close()
1036 logger.debug(output)
1037 return errno.ETIMEDOUT
1039 ### rebooting Dell systems via RAC card
1040 # Marc E. Fiuczynski - June 01 2005
1041 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1044 def runcmd(command, args, username, password, timeout = None):
1047 result_ready = threading.Condition()
1051 result_ready.acquire()
1055 result_ready.notify()
1056 result_ready.release()
1058 def do_command(command, username, password):
1061 # Popen4 is a popen-type class that combines stdout and stderr
1062 p = popen2.Popen4(command)
1064 # read all output data
1065 p.tochild.write("%s\n" % username)
1066 p.tochild.write("%s\n" % password)
1068 data = p.fromchild.read()
1071 # might get interrupted by a signal in poll() or waitpid()
1074 set_result((retval, data))
1077 if ex.errno == errno.EINTR:
1080 except Exception, ex:
1084 command = " ".join([command] + args)
1086 worker = threading.Thread(target = do_command, args = (command, username, password, ))
1087 worker.setDaemon(True)
1088 result_ready.acquire()
1090 result_ready.wait(timeout)
1092 if result == [None]:
1093 raise Exception, "command timed-out: '%s'" % command
1095 result_ready.release()
1098 if isinstance(result, Exception):
1101 (retval, data) = result
1102 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1105 out = "system command ('%s') " % command
1106 if os.WIFEXITED(retval):
1107 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1109 out += "killed by signal %d" % os.WTERMSIG(retval)
1111 out += "; output follows:\n" + data
1112 raise Exception, out
1114 def racadm_reboot(host, username, password, port, dryrun):
1117 ip = socket.gethostbyname(host)
1119 cmd = "/usr/sbin/racadm"
1122 output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1125 output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1128 print "RUNCMD: %s" % output
1130 logger.debug(output)
1133 except Exception, err:
1134 logger.debug("runcmd raised exception %s" % err)
1140 if pcu['hostname'] is not None and pcu['hostname'] is not "":
1141 return pcu['hostname']
1142 elif pcu['ip'] is not None and pcu['ip'] is not "":
1148 fb =soltesz.dbLoad("findbadpcus")
1150 def get_pcu_values(pcu_id):
1151 # TODO: obviously, this shouldn't be loaded each time...
1154 values = fb['nodes']["id_%s" % pcu_id]['values']
1160 def reboot(nodename):
1161 return reboot_policy(nodename, True, False)
1163 def reboot_policy(nodename, continue_probe, dryrun):
1166 pcu = plc.getpcu(nodename)
1168 logger.debug("no pcu for %s" % hostname)
1169 print "no pcu for %s" % hostname
1170 return False # "%s has no pcu" % nodename
1172 values = get_pcu_values(pcu['pcu_id'])
1174 logger.debug("No values for pcu probe %s" % hostname)
1175 print "No values for pcu probe %s" % hostname
1176 return False #"no info for pcu_id %s" % pcu['pcu_id']
1179 logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1181 ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1190 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1194 # DataProbe iPal (many sites)
1195 if continue_probe and values['model'].find("Dataprobe IP-41x/IP-81x") >= 0:
1196 ipal = IPAL(values, verbose, ['23', '80', '9100'])
1197 rb_ret = ipal.reboot(values[nodename], dryrun)
1199 # APC Masterswitch (Berkeley)
1200 elif continue_probe and values['model'].find("APC AP79xx/Masterswitch") >= 0:
1203 # TODO: make a more robust version of APC
1204 if values['pcu_id'] in [1163,1055,1111,1231,1113,1127,1128,1148]:
1205 apc = APCEurope(values, verbose, ['22', '23'])
1206 rb_ret = apc.reboot(values[nodename], dryrun)
1208 elif values['pcu_id'] in [1110,86]:
1209 apc = APCBrazil(values, verbose, ['22', '23'])
1210 rb_ret = apc.reboot(values[nodename], dryrun)
1212 elif values['pcu_id'] in [1221,1225]:
1213 apc = APCBerlin(values, verbose, ['22', '23'])
1214 rb_ret = apc.reboot(values[nodename], dryrun)
1216 elif values['pcu_id'] in [1173,1221,1220]:
1217 apc = APCFolsom(values, verbose, ['22', '23'])
1218 rb_ret = apc.reboot(values[nodename], dryrun)
1221 apc = APCMaster(values, verbose, ['22', '23'])
1222 rb_ret = apc.reboot(values[nodename], dryrun)
1225 elif continue_probe and values['model'].find("Baytech DS4-RPC") >= 0:
1226 if values['pcu_id'] in [1237,1052,1209,1002,1008,1041,1013,1022]:
1227 # These require a 'ctrl-c' to be sent...
1228 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1229 rb_ret = baytech.reboot(values[nodename], dryrun)
1231 elif values['pcu_id'] in [93]:
1232 baytech = BayTechAU(values, verbose, ['22', '23'])
1233 rb_ret = baytech.reboot(values[nodename], dryrun)
1235 elif values['pcu_id'] in [1057]:
1236 # These require a 'ctrl-c' to be sent...
1237 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1238 rb_ret = baytech.reboot(values[nodename], dryrun)
1240 elif values['pcu_id'] in [1012]:
1241 # This pcu sometimes doesn't present the 'Username' prompt,
1242 # unless you immediately try again...
1244 baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1245 rb_ret = baytech.reboot(values[nodename], dryrun)
1247 baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1248 rb_ret = baytech.reboot(values[nodename], dryrun)
1250 baytech = BayTech(values, verbose, ['22', '23'])
1251 rb_ret = baytech.reboot(values[nodename], dryrun)
1254 elif continue_probe and values['model'].find("HP iLO") >= 0:
1256 hpilo = HPiLO(values, verbose, ['22'])
1257 rb_ret = hpilo.reboot(0, dryrun)
1259 hpilo = HPiLOHttps(values, verbose, ['443'])
1260 rb_ret = hpilo.reboot(0, dryrun)
1262 hpilo = HPiLOHttps(values, verbose, ['443'])
1263 rb_ret = hpilo.reboot(0, dryrun)
1266 elif continue_probe and values['model'].find("Dell RAC") >= 0:
1267 # TODO: I don't think DRACRacAdm will throw an exception for the
1268 # default method to catch...
1270 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1271 rb_ret = drac.reboot(0, dryrun)
1273 drac = DRAC(values, verbose, ['22'])
1274 rb_ret = drac.reboot(0, dryrun)
1276 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1277 wti = WTIIPS4(values, verbose, ['23'])
1278 rb_ret = wti.reboot(values[nodename], dryrun)
1280 elif continue_probe and values['model'].find("Intel AMT") >= 0:
1281 amt = IntelAMT(values, verbose, ['16992'])
1282 rb_ret = amt.reboot(values[nodename], dryrun)
1284 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1285 elif continue_probe and \
1286 (values['model'].find("BlackBox PS5xx") >= 0 or
1287 values['model'].find("ePowerSwitch 1/4/8x") >=0 ):
1289 # TODO: allow a different port than http 80.
1290 if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1291 eps = ePowerSwitchGood(values, verbose, ['80'])
1292 elif values['pcu_id'] in [1003]:
1295 eps = ePowerSwitch(values, verbose, ['80'])
1297 eps = ePowerSwitchGood(values, verbose, ['80'])
1299 rb_ret = eps.reboot(values[nodename], dryrun)
1301 elif continue_probe:
1302 rb_ret = "Unsupported_PCU"
1304 elif continue_probe == False:
1305 if 'portstatus' in values:
1312 except ExceptionPort, err:
1317 #elif continue_probe and values['protocol'] == "racadm" and \
1318 # values['model'] == "RAC":
1319 # rb_ret = racadm_reboot(pcu_name(values),
1320 # values['username'],
1321 # values['password'],
1326 logger.setLevel(logging.DEBUG)
1327 ch = logging.StreamHandler()
1328 ch.setLevel(logging.DEBUG)
1329 formatter = logging.Formatter('LOGGER - %(message)s')
1330 ch.setFormatter(formatter)
1331 logger.addHandler(ch)
1334 if "test" in sys.argv:
1339 for node in sys.argv[1:]:
1340 if node == "test": continue
1342 print "Rebooting %s" % node
1343 if reboot_policy(node, True, dryrun):
1347 except Exception, err:
1348 import traceback; traceback.print_exc()
1351 if __name__ == '__main__':
1353 logger = logging.getLogger("monitor")