3 # Reboot specified nodes
9 import errno, time, traceback
11 import threading, popen2
17 from subprocess import PIPE, Popen
18 import ssh.pxssh as pxssh
19 import ssh.pexpect as pexpect
22 # Use our versions of telnetlib and pyssh
23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
31 # Event class ID from pcu events
32 #NODE_POWER_CONTROL = 3
35 #MONITOR_USER_ID = 11142
38 logger = logging.getLogger("monitor")
42 class ExceptionNoTransport(Exception): pass
43 class ExceptionNotFound(Exception): pass
44 class ExceptionPassword(Exception): pass
45 class ExceptionTimeout(Exception): pass
46 class ExceptionPrompt(Exception): pass
47 class ExceptionSequence(Exception): pass
48 class ExceptionReset(Exception): pass
49 class ExceptionPort(Exception): pass
50 class ExceptionUsername(Exception): pass
52 def telnet_answer(telnet, expected, buffer):
55 output = telnet.read_until(expected, TELNET_TIMEOUT)
57 # logger.debug(output)
58 if output.find(expected) == -1:
59 raise ExceptionNotFound, "'%s' not found" % expected
61 telnet.write(buffer + "\r\n")
64 # PCU has model, host, preferred-port, user, passwd,
66 # This is an object derived directly form the PLCAPI DB fields
68 def __init__(self, plc_pcu_dict):
69 for field in ['username', 'password', 'site_id',
72 'node_ids', 'ports', ]:
73 if field in plc_pcu_dict:
74 self.__setattr__(field, plc_pcu_dict[field])
76 raise Exception("No such field %s in PCU object" % field)
78 # These are the convenience functions build around the PCU object.
80 def __init__(self, plc_pcu_dict):
81 PCU.__init__(self, plc_pcu_dict)
82 self.host = self.pcu_name()
85 if self.hostname is not None and self.hostname is not "":
87 elif self.ip is not None and self.ip is not "":
92 def nodeidToPort(self, node_id):
93 if node_id in self.node_ids:
94 for i in range(0, len(self.node_ids)):
95 if node_id == self.node_ids[i]:
98 raise Exception("No such Node ID: %d" % node_id)
100 # This class captures the observed pcu records from FindBadPCUs.py
102 def __init__(self, pcu_record_dict):
103 for field in ['nodenames', 'portstatus',
106 if field in pcu_record_dict:
107 if field == "reboot":
108 self.__setattr__("reboot_str", pcu_record_dict[field])
110 self.__setattr__(field, pcu_record_dict[field])
112 raise Exception("No such field %s in pcu record dict" % field)
121 def __init__(self, type, verbose):
123 self.verbose = verbose
124 self.transport = None
130 def open(self, host, username=None, password=None, prompt="User Name"):
133 if self.type == self.TELNET:
134 transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
135 transport.set_debuglevel(self.verbose)
136 if username is not None:
137 self.transport = transport
138 self.ifThenSend(prompt, username, ExceptionUsername)
140 elif self.type == self.SSH:
141 if username is not None:
142 transport = pyssh.Ssh(username, host)
143 transport.set_debuglevel(self.verbose)
145 # TODO: have an ssh set_debuglevel() also...
147 raise Exception("Username cannot be None for ssh transport.")
148 elif self.type == self.HTTP:
149 self.url = "http://%s:%d/" % (host,80)
150 uri = "%s:%d" % (host,80)
153 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
154 authinfo.add_password (None, uri, username, password)
155 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
157 transport = urllib2.build_opener(authhandler)
160 raise Exception("Unknown transport type: %s" % self.type)
162 self.transport = transport
166 if self.type == self.TELNET:
167 self.transport.close()
168 elif self.type == self.SSH:
169 self.transport.close()
170 elif self.type == self.HTTP:
173 raise Exception("Unknown transport type %s" % self.type)
174 self.transport = None
176 def sendHTTP(self, resource, data):
178 print "POSTing '%s' to %s" % (data,self.url + resource)
181 f = self.transport.open(self.url + resource ,data)
186 except urllib2.URLError,err:
187 logger.info('Could not open http connection', err)
188 return "http transport error"
192 def sendPassword(self, password, prompt=None):
193 if self.type == self.TELNET:
195 self.ifThenSend("Password", password, ExceptionPassword)
197 self.ifThenSend(prompt, password, ExceptionPassword)
198 elif self.type == self.SSH:
199 self.ifThenSend("password:", password, ExceptionPassword)
200 elif self.type == self.HTTP:
203 raise Exception("Unknown transport type: %s" % self.type)
205 def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
207 if self.transport != None:
208 output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
209 if output.find(expected) == -1:
210 raise ErrorClass, "'%s' not found" % expected
212 self.transport.write(buffer + "\r\n")
214 raise ExceptionNoTransport("transport object is type None")
216 def ifElse(self, expected, ErrorClass):
218 self.transport.read_until(expected, self.TELNET_TIMEOUT)
220 raise ErrorClass("Could not find '%s' within timeout" % expected)
223 class PCUControl(Transport,PCUModel,PCURecord):
224 def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
225 PCUModel.__init__(self, plc_pcu_record)
226 PCURecord.__init__(self, plc_pcu_record)
229 if '22' in supported_ports and self.portstatus['22'] == "open":
231 elif '23' in supported_ports and self.portstatus['23'] == "open":
232 type = Transport.TELNET
233 elif '80' in supported_ports and self.portstatus['80'] == "open":
234 type = Transport.HTTP
235 elif '443' in supported_ports and self.portstatus['443'] == "open":
236 type = Transport.HTTP
237 elif '5869' in supported_ports and self.portstatus['5869'] == "open":
238 # For DRAC cards. not sure how much it's used in the
239 # protocol.. but racadm opens this port.
240 type = Transport.HTTP
242 raise ExceptionPort("Unsupported Port: No transport from open ports")
244 raise Exception("No Portstatus: No transport because no open ports")
245 Transport.__init__(self, type, verbose)
247 def run(self, node_port, dryrun):
248 """ This function is to be defined by the specific PCU instance. """
251 def reboot(self, node_port, dryrun):
253 return self.run(node_port, dryrun)
254 except ExceptionNotFound, err:
255 return "error: " + str(err)
256 except ExceptionPassword, err:
257 return "password exception: " + str(err)
258 except ExceptionTimeout, err:
259 return "timeout exception: " + str(err)
260 except ExceptionUsername, err:
261 return "exception: no username prompt: " + str(err)
262 except ExceptionSequence, err:
263 return "sequence error: " + str(err)
264 except ExceptionPrompt, err:
265 return "prompt exception: " + str(err)
266 except ExceptionPort, err:
267 return "no ports exception: " + str(err)
268 except socket.error, err:
269 return "socket error: timeout: " + str(err)
270 except EOFError, err:
272 logger.debug("reboot: EOF")
274 self.transport.close()
276 traceback.print_exc()
277 return "EOF connection reset" + str(err)
278 #except Exception, err:
280 # logger.debug("reboot: Exception")
283 # self.transport.close()
285 # traceback.print_exc()
286 # return "generic exception; unknown problem."
289 class IPAL(PCUControl):
290 def run(self, node_port, dryrun):
293 # XXX Some iPals require you to hit Enter a few times first
294 self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
297 self.ifThenSend("Password >", self.password, ExceptionPassword)
298 self.transport.write("\r\n\r\n")
300 if not dryrun: # P# - Pulse relay
301 self.ifThenSend("Enter >",
304 # Get the next prompt
305 self.ifElse("Enter >", ExceptionTimeout)
310 class APCEurope(PCUControl):
311 def run(self, node_port, dryrun):
312 self.open(self.host, self.username)
313 self.sendPassword(self.password)
315 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
316 self.ifThenSend("\r\n> ", "2")
317 self.ifThenSend("\r\n> ", str(node_port))
318 # 3- Immediate Reboot
319 self.ifThenSend("\r\n> ", "3")
322 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
326 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
327 "", ExceptionSequence)
328 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
333 class APCBrazil(PCUControl):
334 def run(self, node_port, dryrun):
335 self.open(self.host, self.username)
336 self.sendPassword(self.password)
338 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
339 self.ifThenSend("\r\n> ", str(node_port))
340 # 4- Immediate Reboot
341 self.ifThenSend("\r\n> ", "4")
344 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
348 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
349 "", ExceptionSequence)
350 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
355 class APCBerlin(PCUControl):
356 def run(self, node_port, dryrun):
357 self.open(self.host, self.username)
358 self.sendPassword(self.password)
360 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
361 self.ifThenSend("\r\n> ", "2")
362 self.ifThenSend("\r\n> ", "1")
363 self.ifThenSend("\r\n> ", str(node_port))
364 # 3- Immediate Reboot
365 self.ifThenSend("\r\n> ", "3")
368 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
372 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
373 "", ExceptionSequence)
374 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
379 class APCFolsom(PCUControl):
380 def run(self, node_port, dryrun):
381 self.open(self.host, self.username)
382 self.sendPassword(self.password)
384 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
385 self.ifThenSend("\r\n> ", "2")
386 self.ifThenSend("\r\n> ", "1")
387 self.ifThenSend("\r\n> ", str(node_port))
388 self.ifThenSend("\r\n> ", "1")
390 # 3- Immediate Reboot
391 self.ifThenSend("\r\n> ", "3")
394 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
398 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
399 "", ExceptionSequence)
400 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
405 class APCMaster(PCUControl):
406 def run(self, node_port, dryrun):
407 self.open(self.host, self.username)
408 self.sendPassword(self.password)
411 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
412 # 3- Outlet Control/Config
413 self.ifThenSend("\r\n> ", "3")
415 self.ifThenSend("\r\n> ", str(node_port))
417 self.ifThenSend("\r\n> ", "1")
418 # 3- Immediate Reboot
419 self.ifThenSend("\r\n> ", "3")
422 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
426 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
427 "", ExceptionSequence)
428 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
433 class APC(PCUControl):
434 def __init__(self, plc_pcu_record, verbose):
435 PCUControl.__init__(self, plc_pcu_record, verbose)
437 self.master = APCMaster(plc_pcu_record, verbose)
438 self.folsom = APCFolsom(plc_pcu_record, verbose)
439 self.europe = APCEurope(plc_pcu_record, verbose)
441 def run(self, node_port, dryrun):
445 for pcu in [self.master, self.europe, self.folsom]:
448 print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
451 time.sleep(sleep_time)
452 ret = pcu.reboot(node_port, dryrun)
453 except ExceptionSequence, err:
459 return "Unknown reboot sequence for APC PCU"
463 class DRACRacAdm(PCUControl):
464 def run(self, node_port, dryrun):
466 print "trying racadm_reboot..."
467 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
471 class DRAC(PCUControl):
472 def run(self, node_port, dryrun):
473 self.open(self.host, self.username)
474 self.sendPassword(self.password)
476 print "logging in..."
477 self.transport.write("\r\n")
480 self.ifThenSend("[%s]#" % self.username, "getsysinfo")
483 self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
485 self.ifThenSend("[%s]#" % self.username, "exit")
490 class HPiLO(PCUControl):
491 def run(self, node_port, dryrun):
492 self.open(self.host, self.username)
493 self.sendPassword(self.password)
496 self.ifThenSend("</>hpiLO->", "cd system1")
498 # Reboot Outlet N (Y/N)?
500 self.ifThenSend("</system1>hpiLO->", "POWER")
503 self.ifThenSend("</system1>hpiLO->", "reset")
505 self.ifThenSend("</system1>hpiLO->", "exit")
511 class HPiLOHttps(PCUControl):
512 def run(self, node_port, dryrun):
515 #cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s'" % (
516 # self.host, "iloxml/Get_Network.xml",
517 # self.username, self.password)
518 #p_ilo = Popen(cmd, stdout=PIPE, shell=True)
519 #cmd2 = "grep 'MESSAGE' | grep -v 'No error'"
520 #p_grep = Popen(cmd2, stdin=p_ilo.stdout, stdout=PIPE, stderr=PIPE, shell=True)
521 #sout, serr = p_grep.communicate()
523 locfg = soltesz.CMD()
524 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
525 self.host, "iloxml/Get_Network.xml",
526 self.username, self.password)
527 sout, serr = locfg.run_noexcept(cmd)
531 if sout.strip() != "":
532 print "sout: %s" % sout.strip()
536 locfg = soltesz.CMD()
537 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
538 self.host, "iloxml/Reset_Server.xml",
539 self.username, self.password)
540 sout, serr = locfg.run_noexcept(cmd)
542 #cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s'" % (
543 # self.host, "iloxml/Reset_Server.xml",
544 # self.username, self.password)
546 #p_ilo = Popen(cmd, stdin=PIPE, stdout=PIPE, shell=True)
547 #cmd2 = "grep 'MESSAGE' | grep -v 'No error'"
548 #p_grep = Popen(cmd2, shell=True, stdin=p_ilo.stdout, stdout=PIPE, stderr=PIPE)
549 #sout, serr = p_grep.communicate()
552 # print "p_ilo wait failed."
556 # print "p_grep wait failed."
559 if sout.strip() != "":
560 print "sout: %s" % sout.strip()
564 class BayTechAU(PCUControl):
565 def run(self, node_port, dryrun):
566 self.open(self.host, self.username, None, "Enter user name:")
567 self.sendPassword(self.password, "Enter Password:")
569 #self.ifThenSend("RPC-16>", "Status")
570 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
572 # Reboot Outlet N (Y/N)?
574 self.ifThenSend("(Y/N)?", "N")
576 self.ifThenSend("(Y/N)?", "Y")
577 self.ifThenSend("RPC3-NC>", "")
582 class BayTechGeorgeTown(PCUControl):
583 def run(self, node_port, dryrun):
584 self.open(self.host, self.username, None, "Enter user name:")
585 self.sendPassword(self.password, "Enter Password:")
587 #self.ifThenSend("RPC-16>", "Status")
589 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
591 # Reboot Outlet N (Y/N)?
593 self.ifThenSend("(Y/N)?", "N")
595 self.ifThenSend("(Y/N)?", "Y")
596 self.ifThenSend("RPC-16>", "")
601 class BayTechCtrlCUnibe(PCUControl):
603 For some reason, these units let you log in fine, but they hang
604 indefinitely, unless you send a Ctrl-C after the password. No idea
607 def run(self, node_port, dryrun):
608 print "BayTechCtrlC %s" % self.host
610 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
612 if not s.login(self.host, self.username, self.password, ssh_options):
613 raise ExceptionPassword("Invalid Password")
614 # Otherwise, the login succeeded.
616 # Send a ctrl-c to the remote process.
617 print "sending ctrl-c"
620 # Control Outlets (5 ,1).........5
622 index = s.expect(["Enter Request :"])
627 index = s.expect(["DS-RPC>", "Enter user name:"])
629 s.send(self.username + "\r\n")
630 index = s.expect(["DS-RPC>"])
633 print "Reboot %d" % node_port
634 s.send("Reboot %d\r\n" % node_port)
636 index = s.expect(["(Y/N)?"])
645 #index = s.expect(["DS-RPC>"])
646 #print "got prompt back"
651 raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
652 except pexpect.TIMEOUT:
653 raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
657 class BayTechCtrlC(PCUControl):
659 For some reason, these units let you log in fine, but they hang
660 indefinitely, unless you send a Ctrl-C after the password. No idea
663 def run(self, node_port, dryrun):
664 print "BayTechCtrlC %s" % self.host
666 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
668 if not s.login(self.host, self.username, self.password, ssh_options):
669 raise ExceptionPassword("Invalid Password")
670 # Otherwise, the login succeeded.
672 # Send a ctrl-c to the remote process.
673 print "sending ctrl-c"
676 # Control Outlets (5 ,1).........5
678 index = s.expect(["Enter Request :"])
683 index = s.expect(["DS-RPC>", "Enter user name:"])
685 print "sending username"
686 s.send(self.username + "\r\n")
687 index = s.expect(["DS-RPC>"])
690 print "Reboot %d" % node_port
691 s.send("Reboot %d\r\n" % node_port)
693 index = s.expect(["(Y/N)?"])
702 index = s.expect(["DS-RPC>"])
703 #print "got prompt back"
708 raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
709 except pexpect.TIMEOUT:
710 raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
714 class BayTech(PCUControl):
715 def run(self, node_port, dryrun):
716 self.open(self.host, self.username)
717 self.sendPassword(self.password)
719 # Control Outlets (5 ,1).........5
720 self.ifThenSend("Enter Request :", "5")
724 self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
725 except ExceptionNotFound, msg:
726 # one machine is configured to ask for a username,
727 # even after login...
728 print "msg: %s" % msg
729 self.transport.write(self.username + "\r\n")
730 self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
732 # Reboot Outlet N (Y/N)?
734 self.ifThenSend("(Y/N)?", "N")
736 self.ifThenSend("(Y/N)?", "Y")
737 self.ifThenSend("DS-RPC>", "")
742 class WTIIPS4(PCUControl):
743 def run(self, node_port, dryrun):
745 self.sendPassword(self.password, "Enter Password:")
747 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
749 self.ifThenSend("Sure? (Y/N): ", "N")
751 self.ifThenSend("Sure? (Y/N): ", "Y")
753 self.ifThenSend("IPS> ", "")
758 class ePowerSwitchGood(PCUControl):
760 # The old code used Python's HTTPPasswordMgrWithDefaultRealm()
761 # For some reason this both doesn't work and in some cases, actually
762 # hangs the PCU. Definitely not what we want.
764 # The code below is much simpler. Just letting things fail first,
765 # and then, trying again with authentication string in the header.
767 def run(self, node_port, dryrun):
768 self.transport = None
769 self.url = "http://%s:%d/" % (self.host,80)
770 uri = "%s:%d" % (self.host,80)
772 req = urllib2.Request(self.url)
774 handle = urllib2.urlopen(req)
776 # NOTE: this is expected to fail initially
783 return "ERROR: not protected by HTTP authentication"
785 if not hasattr(e, 'code') or e.code != 401:
786 return "ERROR: failed for: %s" % str(e)
788 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
789 # NOTE: assuming basic realm authentication.
790 authheader = "Basic %s" % base64data
791 req.add_header("Authorization", authheader)
794 f = urllib2.urlopen(req)
796 # failing here means the User/passwd is wrong (hopefully)
797 raise ExceptionPassword("Incorrect username/password")
799 # TODO: after verifying that the user/password is correct, we should
800 # actually reboot the given node.
803 # add data to handler,
804 # fetch url one more time on cmd.html, econtrol.html or whatever.
807 if self.verbose: print f.read()
813 class ePowerSwitchOld(PCUControl):
814 def run(self, node_port, dryrun):
815 self.url = "http://%s:%d/" % (self.host,80)
816 uri = "%s:%d" % (self.host,80)
819 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
820 authinfo.add_password (None, uri, self.username, self.password)
821 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
823 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
824 transport = urllib2.build_opener(authinfo)
825 f = transport.open(self.url)
826 if self.verbose: print f.read()
829 transport = urllib2.build_opener(authhandler)
830 f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
831 if self.verbose: print f.read()
836 class ePowerSwitch(PCUControl):
837 def run(self, node_port, dryrun):
838 self.url = "http://%s:%d/" % (self.host,80)
839 uri = "%s:%d" % (self.host,80)
841 # TODO: I'm still not sure what the deal is here.
842 # two independent calls appear to need to be made before the
843 # reboot will succeed. It doesn't seem to be possible to do
844 # this with a single call. I have no idea why.
847 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
848 authinfo.add_password (None, uri, self.username, self.password)
849 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
851 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
852 transport = urllib2.build_opener()
853 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
854 if self.verbose: print f.read()
857 transport = urllib2.build_opener(authhandler)
858 f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
859 if self.verbose: print f.read()
861 # data= "P%d=r" % node_port
862 #self.open(self.host, self.username, self.password)
863 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
864 #self.sendHTTP("econtrol.html", data)
865 #self.sendHTTP("cmd.html", data)
871 ### rebooting european BlackBox PSE boxes
872 # Thierry Parmentelat - May 11 2005
873 # tested on 4-ports models known as PSE505-FR
874 # uses http to POST a data 'P<port>=r'
875 # relies on basic authentication within http1.0
876 # first curl-based script was
877 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
878 # http://<hostname>:<http_port>/cmd.html && echo OK
880 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
884 url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
885 data= "P%d=r" % port_in_pcu
887 logger.debug("POSTing '%s' on %s" % (data,url))
889 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
890 uri = "%s:%d" % (pcu_ip,http_port)
891 authinfo.add_password (None, uri, username, password)
892 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
894 opener = urllib2.build_opener(authhandler)
895 urllib2.install_opener(opener)
901 f = urllib2.urlopen(url,data)
908 except urllib2.URLError,err:
909 logger.info('Could not open http connection', err)
912 ### rebooting x10toggle based systems addressed by port
913 # Marc E. Fiuczynski - May 31 2005
914 # tested on 4-ports models known as PSE505-FR
915 # uses ssh and password to login to an account
916 # that will cause the system to be powercycled.
918 def x10toggle_reboot(ip, username, password, port, dryrun):
923 ssh = pyssh.Ssh(username, ip)
927 telnet_answer(ssh, "password:", password)
931 telnet_answer(ssh, "x10toggle>", "A%d" % port)
939 except Exception, err:
946 return errno.ETIMEDOUT
948 ### rebooting Dell systems via RAC card
949 # Marc E. Fiuczynski - June 01 2005
950 # tested with David Lowenthal's itchy/scratchy nodes at UGA
953 def runcmd(command, args, username, password, timeout = None):
956 result_ready = threading.Condition()
960 result_ready.acquire()
964 result_ready.notify()
965 result_ready.release()
967 def do_command(command, username, password):
970 # Popen4 is a popen-type class that combines stdout and stderr
971 p = popen2.Popen4(command)
973 # read all output data
974 p.tochild.write("%s\n" % username)
975 p.tochild.write("%s\n" % password)
977 data = p.fromchild.read()
980 # might get interrupted by a signal in poll() or waitpid()
983 set_result((retval, data))
986 if ex.errno == errno.EINTR:
989 except Exception, ex:
993 command = " ".join([command] + args)
995 worker = threading.Thread(target = do_command, args = (command, username, password, ))
996 worker.setDaemon(True)
997 result_ready.acquire()
999 result_ready.wait(timeout)
1001 if result == [None]:
1002 raise Exception, "command timed-out: '%s'" % command
1004 result_ready.release()
1007 if isinstance(result, Exception):
1010 (retval, data) = result
1011 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1014 out = "system command ('%s') " % command
1015 if os.WIFEXITED(retval):
1016 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1018 out += "killed by signal %d" % os.WTERMSIG(retval)
1020 out += "; output follows:\n" + data
1021 raise Exception, out
1023 def racadm_reboot(host, username, password, port, dryrun):
1026 ip = socket.gethostbyname(host)
1028 cmd = "/usr/sbin/racadm"
1031 output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1034 output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1037 print "RUNCMD: %s" % output
1039 logger.debug(output)
1042 except Exception, err:
1043 logger.debug("runcmd raised exception %s" % err)
1049 if pcu['hostname'] is not None and pcu['hostname'] is not "":
1050 return pcu['hostname']
1051 elif pcu['ip'] is not None and pcu['ip'] is not "":
1056 def get_pcu_values(pcu_id):
1057 # TODO: obviously, this shouldn't be loaded each time...
1059 fb =soltesz.dbLoad("findbadpcus")
1062 values = fb['nodes']["id_%s" % pcu_id]['values']
1068 def reboot(nodename):
1069 return reboot_policy(nodename, True, False)
1071 def reboot_policy(nodename, continue_probe, dryrun):
1074 pcu = plc.getpcu(nodename)
1076 return False # "%s has no pcu" % nodename
1078 values = get_pcu_values(pcu['pcu_id'])
1080 return False #"no info for pcu_id %s" % pcu['pcu_id']
1083 logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1085 ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1093 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1097 # DataProbe iPal (many sites)
1098 if continue_probe and values['model'].find("Dataprobe IP-41x/IP-81x") >= 0:
1099 ipal = IPAL(values, verbose, ['23'])
1100 rb_ret = ipal.reboot(values[nodename], dryrun)
1102 # APC Masterswitch (Berkeley)
1103 elif continue_probe and values['model'].find("APC AP79xx/Masterswitch") >= 0:
1105 # TODO: make a more robust version of APC
1106 if values['pcu_id'] in [1163,1055,1111,1231,1113,1127,1128,1148]:
1107 apc = APCEurope(values, verbose, ['22', '23'])
1108 rb_ret = apc.reboot(values[nodename], dryrun)
1110 elif values['pcu_id'] in [1110,86]:
1111 apc = APCBrazil(values, verbose, ['22', '23'])
1112 rb_ret = apc.reboot(values[nodename], dryrun)
1114 elif values['pcu_id'] in [1221,1225]:
1115 apc = APCBerlin(values, verbose, ['22', '23'])
1116 rb_ret = apc.reboot(values[nodename], dryrun)
1118 elif values['pcu_id'] in [1173,1221,1220]:
1119 apc = APCFolsom(values, verbose, ['22', '23'])
1120 rb_ret = apc.reboot(values[nodename], dryrun)
1123 apc = APCMaster(values, verbose, ['22', '23'])
1124 rb_ret = apc.reboot(values[nodename], dryrun)
1127 elif continue_probe and values['model'].find("Baytech DS4-RPC") >= 0:
1128 if values['pcu_id'] in [1237,1052,1209,1002,1008,1041,1013,1022]:
1129 # These require a 'ctrl-c' to be sent...
1130 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1131 rb_ret = baytech.reboot(values[nodename], dryrun)
1133 elif values['pcu_id'] in [93]:
1134 baytech = BayTechAU(values, verbose, ['22', '23'])
1135 rb_ret = baytech.reboot(values[nodename], dryrun)
1137 elif values['pcu_id'] in [1057]:
1138 # These require a 'ctrl-c' to be sent...
1139 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1140 rb_ret = baytech.reboot(values[nodename], dryrun)
1142 elif values['pcu_id'] in [1012]:
1143 # This pcu sometimes doesn't present the 'Username' prompt,
1144 # unless you immediately try again...
1146 baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1147 rb_ret = baytech.reboot(values[nodename], dryrun)
1149 baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1150 rb_ret = baytech.reboot(values[nodename], dryrun)
1152 baytech = BayTech(values, verbose, ['22', '23'])
1153 rb_ret = baytech.reboot(values[nodename], dryrun)
1156 elif continue_probe and values['model'].find("HP iLO") >= 0:
1158 hpilo = HPiLO(values, verbose, ['22'])
1159 rb_ret = hpilo.reboot(0, dryrun)
1161 hpilo = HPiLOHttps(values, verbose, ['443'])
1162 rb_ret = hpilo.reboot(0, dryrun)
1164 hpilo = HPiLOHttps(values, verbose, ['443'])
1165 rb_ret = hpilo.reboot(0, dryrun)
1168 elif continue_probe and values['model'].find("Dell RAC") >= 0:
1169 # TODO: I don't think DRACRacAdm will throw an exception for the
1170 # default method to catch...
1172 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1173 rb_ret = drac.reboot(0, dryrun)
1175 drac = DRAC(values, verbose, ['22'])
1176 rb_ret = drac.reboot(0, dryrun)
1178 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1179 wti = WTIIPS4(values, verbose, ['23'])
1180 rb_ret = wti.reboot(values[nodename], dryrun)
1182 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1183 elif continue_probe and \
1184 (values['model'].find("BlackBox PS5xx") >= 0 or
1185 values['model'].find("ePowerSwitch 1/4/8x") >=0 ):
1187 # TODO: allow a different port than http 80.
1188 if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1189 eps = ePowerSwitchGood(values, verbose, ['80'])
1190 elif values['pcu_id'] in [1003]:
1191 eps = ePowerSwitch(values, verbose, ['80'])
1193 eps = ePowerSwitchGood(values, verbose, ['80'])
1195 rb_ret = eps.reboot(values[nodename], dryrun)
1197 elif continue_probe:
1198 rb_ret = "Unsupported_PCU"
1200 elif continue_probe == False:
1201 if 'portstatus' in values:
1208 except ExceptionPort, err:
1213 #elif continue_probe and values['protocol'] == "racadm" and \
1214 # values['model'] == "RAC":
1215 # rb_ret = racadm_reboot(pcu_name(values),
1216 # values['username'],
1217 # values['password'],
1222 logger.setLevel(logging.DEBUG)
1223 ch = logging.StreamHandler()
1224 ch.setLevel(logging.DEBUG)
1225 formatter = logging.Formatter('LOGGER - %(message)s')
1226 ch.setFormatter(formatter)
1227 logger.addHandler(ch)
1230 for node in sys.argv[1:]:
1231 print "Rebooting %s" % node
1232 if reboot_policy(node, True, False):
1236 except Exception, err:
1239 if __name__ == '__main__':
1241 logger = logging.getLogger("monitor")