3 # Reboot specified nodes
9 import errno, time, traceback
11 import threading, popen2
17 from subprocess import PIPE, Popen
18 import ssh.pxssh as pxssh
19 import ssh.pexpect as pexpect
21 plc_lock = threading.Lock()
23 # Use our versions of telnetlib and pyssh
24 sys.path.insert(0, os.path.dirname(sys.argv[0]))
26 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
32 # Event class ID from pcu events
33 #NODE_POWER_CONTROL = 3
36 #MONITOR_USER_ID = 11142
39 logger = logging.getLogger("monitor")
43 class ExceptionNoTransport(Exception): pass
44 class ExceptionNotFound(Exception): pass
45 class ExceptionPassword(Exception): pass
46 class ExceptionTimeout(Exception): pass
47 class ExceptionPrompt(Exception): pass
48 class ExceptionSequence(Exception): pass
49 class ExceptionReset(Exception): pass
50 class ExceptionPort(Exception): pass
51 class ExceptionUsername(Exception): pass
53 def telnet_answer(telnet, expected, buffer):
56 output = telnet.read_until(expected, TELNET_TIMEOUT)
58 # logger.debug(output)
59 if output.find(expected) == -1:
60 raise ExceptionNotFound, "'%s' not found" % expected
62 telnet.write(buffer + "\r\n")
65 # PCU has model, host, preferred-port, user, passwd,
67 # This is an object derived directly form the PLCAPI DB fields
69 def __init__(self, plc_pcu_dict):
70 for field in ['username', 'password', 'site_id',
73 'node_ids', 'ports', ]:
74 if field in plc_pcu_dict:
75 self.__setattr__(field, plc_pcu_dict[field])
77 raise Exception("No such field %s in PCU object" % field)
79 # These are the convenience functions build around the PCU object.
81 def __init__(self, plc_pcu_dict):
82 PCU.__init__(self, plc_pcu_dict)
83 self.host = self.pcu_name()
86 if self.hostname is not None and self.hostname is not "":
88 elif self.ip is not None and self.ip is not "":
93 def nodeidToPort(self, node_id):
94 if node_id in self.node_ids:
95 for i in range(0, len(self.node_ids)):
96 if node_id == self.node_ids[i]:
99 raise Exception("No such Node ID: %d" % node_id)
101 # This class captures the observed pcu records from FindBadPCUs.py
103 def __init__(self, pcu_record_dict):
104 for field in ['nodenames', 'portstatus',
107 if field in pcu_record_dict:
108 if field == "reboot":
109 self.__setattr__("reboot_str", pcu_record_dict[field])
111 self.__setattr__(field, pcu_record_dict[field])
113 raise Exception("No such field %s in pcu record dict" % field)
122 def __init__(self, type, verbose):
124 self.verbose = verbose
125 self.transport = None
131 def open(self, host, username=None, password=None, prompt="User Name"):
134 if self.type == self.TELNET:
135 transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
136 transport.set_debuglevel(self.verbose)
137 if username is not None:
138 self.transport = transport
139 self.ifThenSend(prompt, username, ExceptionUsername)
141 elif self.type == self.SSH:
142 if username is not None:
143 transport = pyssh.Ssh(username, host)
144 transport.set_debuglevel(self.verbose)
146 # TODO: have an ssh set_debuglevel() also...
148 raise Exception("Username cannot be None for ssh transport.")
149 elif self.type == self.HTTP:
150 self.url = "http://%s:%d/" % (host,80)
151 uri = "%s:%d" % (host,80)
154 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
155 authinfo.add_password (None, uri, username, password)
156 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
158 transport = urllib2.build_opener(authhandler)
161 raise Exception("Unknown transport type: %s" % self.type)
163 self.transport = transport
167 if self.type == self.TELNET:
168 self.transport.close()
169 elif self.type == self.SSH:
170 self.transport.close()
171 elif self.type == self.HTTP:
174 raise Exception("Unknown transport type %s" % self.type)
175 self.transport = None
177 def sendHTTP(self, resource, data):
179 print "POSTing '%s' to %s" % (data,self.url + resource)
182 f = self.transport.open(self.url + resource ,data)
187 except urllib2.URLError,err:
188 logger.info('Could not open http connection', err)
189 return "http transport error"
193 def sendPassword(self, password, prompt=None):
194 if self.type == self.TELNET:
196 self.ifThenSend("Password", password, ExceptionPassword)
198 self.ifThenSend(prompt, password, ExceptionPassword)
199 elif self.type == self.SSH:
200 self.ifThenSend("password:", password, ExceptionPassword)
201 elif self.type == self.HTTP:
204 raise Exception("Unknown transport type: %s" % self.type)
206 def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
208 if self.transport != None:
209 output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
210 if output.find(expected) == -1:
211 raise ErrorClass, "'%s' not found" % expected
213 self.transport.write(buffer + "\r\n")
215 raise ExceptionNoTransport("transport object is type None")
217 def ifElse(self, expected, ErrorClass):
219 self.transport.read_until(expected, self.TELNET_TIMEOUT)
221 raise ErrorClass("Could not find '%s' within timeout" % expected)
224 class PCUControl(Transport,PCUModel,PCURecord):
225 def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
226 PCUModel.__init__(self, plc_pcu_record)
227 PCURecord.__init__(self, plc_pcu_record)
230 if '22' in supported_ports and self.portstatus['22'] == "open":
232 elif '23' in supported_ports and self.portstatus['23'] == "open":
233 type = Transport.TELNET
234 elif '80' in supported_ports and self.portstatus['80'] == "open":
235 type = Transport.HTTP
236 elif '443' in supported_ports and self.portstatus['443'] == "open":
237 type = Transport.HTTP
238 elif '5869' in supported_ports and self.portstatus['5869'] == "open":
239 # For DRAC cards. not sure how much it's used in the
240 # protocol.. but racadm opens this port.
241 type = Transport.HTTP
243 raise ExceptionPort("Unsupported Port: No transport from open ports")
244 Transport.__init__(self, type, verbose)
246 def run(self, node_port, dryrun):
247 """ This function is to be defined by the specific PCU instance. """
250 def reboot(self, node_port, dryrun):
252 return self.run(node_port, dryrun)
253 except ExceptionNotFound, err:
254 return "error: " + str(err)
255 except ExceptionPassword, err:
256 return "password exception: " + str(err)
257 except ExceptionTimeout, err:
258 return "timeout exception: " + str(err)
259 except ExceptionUsername, err:
260 return "exception: no username prompt: " + str(err)
261 except ExceptionSequence, err:
262 return "sequence error: " + str(err)
263 except ExceptionPrompt, err:
264 return "prompt exception: " + str(err)
265 except ExceptionPort, err:
266 return "no ports exception: " + str(err)
267 except socket.error, err:
268 return "socket error: timeout: " + str(err)
269 except EOFError, err:
271 logger.debug("reboot: EOF")
273 self.transport.close()
275 traceback.print_exc()
276 return "EOF connection reset" + str(err)
277 #except Exception, err:
279 # logger.debug("reboot: Exception")
282 # self.transport.close()
284 # traceback.print_exc()
285 # return "generic exception; unknown problem."
288 class IPAL(PCUControl):
289 def run(self, node_port, dryrun):
292 # XXX Some iPals require you to hit Enter a few times first
293 self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
296 self.ifThenSend("Password >", self.password, ExceptionPassword)
297 self.transport.write("\r\n\r\n")
299 if not dryrun: # P# - Pulse relay
300 self.ifThenSend("Enter >",
303 # Get the next prompt
304 self.ifElse("Enter >", ExceptionTimeout)
309 def ipal_reboot(ip, password, port, dryrun):
316 #print "lock acquired"
319 #telnet = telnetlib.Telnet(ip) # , timeout=TELNET_TIMEOUT)
320 telnet = telnetlib.Telnet(ip, timeout=TELNET_TIMEOUT)
323 # traceback.print_exc()
326 telnet.set_debuglevel(verbose)
328 # XXX Some iPals require you to hit Enter a few times first
329 telnet_answer(telnet, "Password >", "\r\n\r\n")
332 telnet_answer(telnet, "Password >", password)
334 # XXX Some iPals require you to hit Enter a few times first
335 telnet.write("\r\n\r\n")
339 telnet_answer(telnet, "Enter >", "P%d" % port)
341 telnet.read_until("Enter >", TELNET_TIMEOUT)
346 #print "lock released"
350 except EOFError, err:
352 logger.debug("ipal_reboot: EOF")
356 traceback.print_exc()
357 #print "lock released"
359 return errno.ECONNRESET
360 except socket.error, err:
361 logger.debug("ipal_reboot: Socket Error")
364 traceback.print_exc()
366 return errno.ETIMEDOUT
368 except Exception, err:
370 logger.debug("ipal_reboot: Exception")
375 traceback.print_exc()
376 #print "lock released"
380 class APCEurope(PCUControl):
381 def run(self, node_port, dryrun):
382 self.open(self.host, self.username)
383 self.sendPassword(self.password)
385 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
386 self.ifThenSend("\r\n> ", "2")
387 self.ifThenSend("\r\n> ", str(node_port))
388 # 3- Immediate Reboot
389 self.ifThenSend("\r\n> ", "3")
392 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
396 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
397 "", ExceptionSequence)
398 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
403 class APCBrazil(PCUControl):
404 def run(self, node_port, dryrun):
405 self.open(self.host, self.username)
406 self.sendPassword(self.password)
408 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
409 self.ifThenSend("\r\n> ", str(node_port))
410 # 4- Immediate Reboot
411 self.ifThenSend("\r\n> ", "4")
414 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
418 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
419 "", ExceptionSequence)
420 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
425 class APCBerlin(PCUControl):
426 def run(self, node_port, dryrun):
427 self.open(self.host, self.username)
428 self.sendPassword(self.password)
430 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
431 self.ifThenSend("\r\n> ", "2")
432 self.ifThenSend("\r\n> ", "1")
433 self.ifThenSend("\r\n> ", str(node_port))
434 # 3- Immediate Reboot
435 self.ifThenSend("\r\n> ", "3")
438 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
442 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
443 "", ExceptionSequence)
444 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
449 class APCFolsom(PCUControl):
450 def run(self, node_port, dryrun):
451 self.open(self.host, self.username)
452 self.sendPassword(self.password)
454 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
455 self.ifThenSend("\r\n> ", "2")
456 self.ifThenSend("\r\n> ", "1")
457 self.ifThenSend("\r\n> ", str(node_port))
458 self.ifThenSend("\r\n> ", "1")
460 # 3- Immediate Reboot
461 self.ifThenSend("\r\n> ", "3")
464 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
468 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
469 "", ExceptionSequence)
470 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
475 class APCMaster(PCUControl):
476 def run(self, node_port, dryrun):
477 self.open(self.host, self.username)
478 self.sendPassword(self.password)
481 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
482 # 3- Outlet Control/Config
483 self.ifThenSend("\r\n> ", "3")
485 self.ifThenSend("\r\n> ", str(node_port))
487 self.ifThenSend("\r\n> ", "1")
488 # 3- Immediate Reboot
489 self.ifThenSend("\r\n> ", "3")
492 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
496 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
497 "", ExceptionSequence)
498 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
503 class APC(PCUControl):
504 def __init__(self, plc_pcu_record, verbose):
505 PCUControl.__init__(self, plc_pcu_record, verbose)
507 self.master = APCMaster(plc_pcu_record, verbose)
508 self.folsom = APCFolsom(plc_pcu_record, verbose)
509 self.europe = APCEurope(plc_pcu_record, verbose)
511 def run(self, node_port, dryrun):
515 for pcu in [self.master, self.europe, self.folsom]:
518 print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
521 time.sleep(sleep_time)
522 ret = pcu.reboot(node_port, dryrun)
523 except ExceptionSequence, err:
529 return "Unknown reboot sequence for APC PCU"
533 class DRACRacAdm(PCUControl):
534 def run(self, node_port, dryrun):
536 print "trying racadm_reboot..."
537 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
541 class DRAC(PCUControl):
542 def run(self, node_port, dryrun):
543 self.open(self.host, self.username)
544 self.sendPassword(self.password)
546 print "logging in..."
547 self.transport.write("\r\n")
550 self.ifThenSend("[%s]#" % self.username, "getsysinfo")
553 self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
555 self.ifThenSend("[%s]#" % self.username, "exit")
560 class HPiLO(PCUControl):
561 def run(self, node_port, dryrun):
562 self.open(self.host, self.username)
563 self.sendPassword(self.password)
566 self.ifThenSend("</>hpiLO->", "cd system1")
568 # Reboot Outlet N (Y/N)?
570 self.ifThenSend("</system1>hpiLO->", "POWER")
573 self.ifThenSend("</system1>hpiLO->", "reset")
575 self.ifThenSend("</system1>hpiLO->", "exit")
581 class HPiLOHttps(PCUControl):
582 def run(self, node_port, dryrun):
584 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p %s" % (
585 self.host, "iloxml/Get_Network.xml",
586 self.username, self.password)
587 p_ilo = Popen(cmd, stdout=PIPE, shell=True)
588 cmd2 = "grep 'MESSAGE' | grep -v 'No error'"
589 p_grep = Popen(cmd2, stdin=p_ilo.stdout, stdout=PIPE, stderr=PIPE, shell=True)
590 sout, serr = p_grep.communicate()
594 if sout.strip() != "":
595 print "sout: %s" % sout.strip()
599 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p %s" % (
600 self.host, "iloxml/Reset_Server.xml",
601 self.username, self.password)
602 p_ilo = Popen(cmd, stdin=PIPE, stdout=PIPE, shell=True)
603 cmd2 = "grep 'MESSAGE' | grep -v 'No error'"
604 p_grep = Popen(cmd2, stdin=p_ilo.stdout, stdout=PIPE, stderr=PIPE)
605 sout, serr = p_grep.communicate()
609 if sout.strip() != "":
610 print "sout: %s" % sout.strip()
615 class BayTechAU(PCUControl):
616 def run(self, node_port, dryrun):
617 self.open(self.host, self.username, None, "Enter user name:")
618 self.sendPassword(self.password, "Enter Password:")
620 #self.ifThenSend("RPC-16>", "Status")
621 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
623 # Reboot Outlet N (Y/N)?
625 self.ifThenSend("(Y/N)?", "N")
627 self.ifThenSend("(Y/N)?", "Y")
628 self.ifThenSend("RPC3-NC>", "")
633 class BayTechGeorgeTown(PCUControl):
634 def run(self, node_port, dryrun):
635 self.open(self.host, self.username, None, "Enter user name:")
636 self.sendPassword(self.password, "Enter Password:")
638 #self.ifThenSend("RPC-16>", "Status")
640 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
642 # Reboot Outlet N (Y/N)?
644 self.ifThenSend("(Y/N)?", "N")
646 self.ifThenSend("(Y/N)?", "Y")
647 self.ifThenSend("RPC-16>", "")
652 class BayTechCtrlCUnibe(PCUControl):
654 For some reason, these units let you log in fine, but they hang
655 indefinitely, unless you send a Ctrl-C after the password. No idea
658 def run(self, node_port, dryrun):
659 print "BayTechCtrlC %s" % self.host
661 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
663 if not s.login(self.host, self.username, self.password, ssh_options):
664 raise ExceptionPassword("Invalid Password")
665 # Otherwise, the login succeeded.
667 # Send a ctrl-c to the remote process.
668 print "sending ctrl-c"
671 # Control Outlets (5 ,1).........5
673 index = s.expect(["Enter Request :"])
678 index = s.expect(["DS-RPC>", "Enter user name:"])
680 s.send(self.username + "\r\n")
681 index = s.expect(["DS-RPC>"])
684 print "Reboot %d" % node_port
685 s.send("Reboot %d\r\n" % node_port)
687 index = s.expect(["(Y/N)?"])
696 #index = s.expect(["DS-RPC>"])
697 #print "got prompt back"
702 raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
703 except pexpect.TIMEOUT:
704 raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
708 class BayTechCtrlC(PCUControl):
710 For some reason, these units let you log in fine, but they hang
711 indefinitely, unless you send a Ctrl-C after the password. No idea
714 def run(self, node_port, dryrun):
715 print "BayTechCtrlC %s" % self.host
717 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
719 if not s.login(self.host, self.username, self.password, ssh_options):
720 raise ExceptionPassword("Invalid Password")
721 # Otherwise, the login succeeded.
723 # Send a ctrl-c to the remote process.
724 print "sending ctrl-c"
727 # Control Outlets (5 ,1).........5
729 index = s.expect(["Enter Request :"])
734 index = s.expect(["DS-RPC>", "Enter user name:"])
736 print "sending username"
737 s.send(self.username + "\r\n")
738 index = s.expect(["DS-RPC>"])
741 print "Reboot %d" % node_port
742 s.send("Reboot %d\r\n" % node_port)
744 index = s.expect(["(Y/N)?"])
753 #index = s.expect(["DS-RPC>"])
754 #print "got prompt back"
759 raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
760 except pexpect.TIMEOUT:
761 raise ExceptionPrompt("Timeout before 'Enter Request' Prompt")
765 class BayTech(PCUControl):
766 def run(self, node_port, dryrun):
767 self.open(self.host, self.username)
768 self.sendPassword(self.password)
770 # Control Outlets (5 ,1).........5
771 self.ifThenSend("Enter Request :", "5")
775 self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
776 except ExceptionNotFound, msg:
777 # one machine is configured to ask for a username,
778 # even after login...
779 print "msg: %s" % msg
780 self.transport.write(self.username + "\r\n")
781 self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
783 # Reboot Outlet N (Y/N)?
785 self.ifThenSend("(Y/N)?", "N")
787 self.ifThenSend("(Y/N)?", "Y")
788 self.ifThenSend("DS-RPC>", "")
793 class WTIIPS4(PCUControl):
794 def run(self, node_port, dryrun):
796 self.sendPassword(self.password, "Enter Password:")
798 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
800 self.ifThenSend("Sure? (Y/N): ", "N")
802 self.ifThenSend("Sure? (Y/N): ", "Y")
804 self.ifThenSend("IPS> ", "")
809 class ePowerSwitchGood(PCUControl):
811 # The old code used Python's HTTPPasswordMgrWithDefaultRealm()
812 # For some reason this both doesn't work and in some cases, actually
813 # hangs the PCU. Definitely not what we want.
815 # The code below is much simpler. Just letting things fail first,
816 # and then, trying again with authentication string in the header.
818 def run(self, node_port, dryrun):
819 self.transport = None
820 self.url = "http://%s:%d/" % (self.host,80)
821 uri = "%s:%d" % (self.host,80)
823 req = urllib2.Request(self.url)
825 handle = urllib2.urlopen(req)
827 # NOTE: this is expected to fail initially
834 return "ERROR: not protected by HTTP authentication"
836 if not hasattr(e, 'code') or e.code != 401:
837 return "ERROR: failed for: %s" % str(e)
839 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
840 # NOTE: assuming basic realm authentication.
841 authheader = "Basic %s" % base64data
842 req.add_header("Authorization", authheader)
845 f = urllib2.urlopen(req)
847 # failing here means the User/passwd is wrong (hopefully)
848 raise ExceptionPassword("Incorrect username/password")
850 # TODO: after verifying that the user/password is correct, we should
851 # actually reboot the given node.
854 # add data to handler,
855 # fetch url one more time on cmd.html, econtrol.html or whatever.
858 if self.verbose: print f.read()
864 class ePowerSwitchOld(PCUControl):
865 def run(self, node_port, dryrun):
866 self.url = "http://%s:%d/" % (self.host,80)
867 uri = "%s:%d" % (self.host,80)
870 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
871 authinfo.add_password (None, uri, self.username, self.password)
872 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
874 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
875 transport = urllib2.build_opener(authinfo)
876 f = transport.open(self.url)
877 if self.verbose: print f.read()
880 transport = urllib2.build_opener(authhandler)
881 f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
882 if self.verbose: print f.read()
887 class ePowerSwitch(PCUControl):
888 def run(self, node_port, dryrun):
889 self.url = "http://%s:%d/" % (self.host,80)
890 uri = "%s:%d" % (self.host,80)
892 # TODO: I'm still not sure what the deal is here.
893 # two independent calls appear to need to be made before the
894 # reboot will succeed. It doesn't seem to be possible to do
895 # this with a single call. I have no idea why.
898 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
899 authinfo.add_password (None, uri, self.username, self.password)
900 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
902 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
903 transport = urllib2.build_opener()
904 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
905 if self.verbose: print f.read()
908 transport = urllib2.build_opener(authhandler)
909 f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
910 if self.verbose: print f.read()
912 # data= "P%d=r" % node_port
913 #self.open(self.host, self.username, self.password)
914 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
915 #self.sendHTTP("econtrol.html", data)
916 #self.sendHTTP("cmd.html", data)
922 ### rebooting european BlackBox PSE boxes
923 # Thierry Parmentelat - May 11 2005
924 # tested on 4-ports models known as PSE505-FR
925 # uses http to POST a data 'P<port>=r'
926 # relies on basic authentication within http1.0
927 # first curl-based script was
928 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
929 # http://<hostname>:<http_port>/cmd.html && echo OK
931 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
935 url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
936 data= "P%d=r" % port_in_pcu
938 logger.debug("POSTing '%s' on %s" % (data,url))
940 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
941 uri = "%s:%d" % (pcu_ip,http_port)
942 authinfo.add_password (None, uri, username, password)
943 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
945 opener = urllib2.build_opener(authhandler)
946 urllib2.install_opener(opener)
952 f = urllib2.urlopen(url,data)
959 except urllib2.URLError,err:
960 logger.info('Could not open http connection', err)
963 ### rebooting x10toggle based systems addressed by port
964 # Marc E. Fiuczynski - May 31 2005
965 # tested on 4-ports models known as PSE505-FR
966 # uses ssh and password to login to an account
967 # that will cause the system to be powercycled.
969 def x10toggle_reboot(ip, username, password, port, dryrun):
974 ssh = pyssh.Ssh(username, ip)
978 telnet_answer(ssh, "password:", password)
982 telnet_answer(ssh, "x10toggle>", "A%d" % port)
990 except Exception, err:
997 return errno.ETIMEDOUT
999 ### rebooting Dell systems via RAC card
1000 # Marc E. Fiuczynski - June 01 2005
1001 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1004 def runcmd(command, args, username, password, timeout = None):
1007 result_ready = threading.Condition()
1011 result_ready.acquire()
1015 result_ready.notify()
1016 result_ready.release()
1018 def do_command(command, username, password):
1021 # Popen4 is a popen-type class that combines stdout and stderr
1022 p = popen2.Popen4(command)
1024 # read all output data
1025 p.tochild.write("%s\n" % username)
1026 p.tochild.write("%s\n" % password)
1028 data = p.fromchild.read()
1031 # might get interrupted by a signal in poll() or waitpid()
1034 set_result((retval, data))
1037 if ex.errno == errno.EINTR:
1040 except Exception, ex:
1044 command = " ".join([command] + args)
1046 worker = threading.Thread(target = do_command, args = (command, username, password, ))
1047 worker.setDaemon(True)
1048 result_ready.acquire()
1050 result_ready.wait(timeout)
1052 if result == [None]:
1053 raise Exception, "command timed-out: '%s'" % command
1055 result_ready.release()
1058 if isinstance(result, Exception):
1061 (retval, data) = result
1062 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1065 out = "system command ('%s') " % command
1066 if os.WIFEXITED(retval):
1067 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1069 out += "killed by signal %d" % os.WTERMSIG(retval)
1071 out += "; output follows:\n" + data
1072 raise Exception, out
1074 def racadm_reboot(ip, username, password, port, dryrun):
1078 cmd = "/usr/sbin/racadm"
1081 output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1084 output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1087 print "RUNCMD: %s" % output
1089 logger.debug(output)
1092 except Exception, err:
1093 logger.debug("runcmd raised exception %s" % err)
1099 if pcu['hostname'] is not None and pcu['hostname'] is not "":
1100 return pcu['hostname']
1101 elif pcu['ip'] is not None and pcu['ip'] is not "":
1106 def get_pcu_values(pcu_id):
1107 # TODO: obviously, this shouldn't be loaded each time...
1109 fb =soltesz.dbLoad("findbadpcus")
1112 values = fb['nodes']["id_%s" % pcu_id]['values']
1118 def check_open_port(values, port_list):
1121 if 'portstatus' in values:
1122 for port in port_list:
1123 if port in values['portstatus'] and \
1124 values['portstatus'][port] == "open":
1130 def reboot_policy(nodename, continue_probe, dryrun):
1133 pcu = plc.getpcu(nodename)
1135 return False # "%s has no pcu" % nodename
1137 values = get_pcu_values(pcu['pcu_id'])
1139 return False #"no info for pcu_id %s" % pcu['pcu_id']
1142 logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1144 ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1151 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1155 # DataProbe iPal (many sites)
1156 if continue_probe and values['model'].find("Dataprobe IP-41x/IP-81x") >= 0:
1157 ipal = IPAL(values, verbose, ['23'])
1158 rb_ret = ipal.reboot(values[nodename], dryrun)
1160 # APC Masterswitch (Berkeley)
1161 elif continue_probe and values['model'].find("APC AP79xx/Masterswitch") >= 0:
1163 # TODO: make a more robust version of APC
1164 if values['pcu_id'] in [1163,1055,1111,1231,1113,1127,1128,1148]:
1165 apc = APCEurope(values, verbose, ['22', '23'])
1166 rb_ret = apc.reboot(values[nodename], dryrun)
1168 elif values['pcu_id'] in [1110,86]:
1169 apc = APCBrazil(values, verbose, ['22', '23'])
1170 rb_ret = apc.reboot(values[nodename], dryrun)
1172 elif values['pcu_id'] in [1221]:
1173 apc = APCBerlin(values, verbose, ['22', '23'])
1174 rb_ret = apc.reboot(values[nodename], dryrun)
1176 elif values['pcu_id'] in [1173,1221,1220,1225]:
1177 apc = APCFolsom(values, verbose, ['22', '23'])
1178 rb_ret = apc.reboot(values[nodename], dryrun)
1181 apc = APCMaster(values, verbose, ['22', '23'])
1182 rb_ret = apc.reboot(values[nodename], dryrun)
1185 elif continue_probe and values['model'].find("Baytech DS4-RPC") >= 0:
1186 if values['pcu_id'] in [1052,1209,1002,1008,1041,1013,1022]:
1187 # These require a 'ctrl-c' to be sent...
1188 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1189 rb_ret = baytech.reboot(values[nodename], dryrun)
1191 elif values['pcu_id'] in [93]:
1192 baytech = BayTechAU(values, verbose, ['22', '23'])
1193 rb_ret = baytech.reboot(values[nodename], dryrun)
1195 elif values['pcu_id'] in [1057]:
1196 # These require a 'ctrl-c' to be sent...
1197 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1198 rb_ret = baytech.reboot(values[nodename], dryrun)
1200 elif values['pcu_id'] in [1012]:
1201 # This pcu sometimes doesn't present the 'Username' prompt,
1202 # unless you immediately try again...
1204 baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1205 rb_ret = baytech.reboot(values[nodename], dryrun)
1207 baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1208 rb_ret = baytech.reboot(values[nodename], dryrun)
1210 baytech = BayTech(values, verbose, ['22', '23'])
1211 rb_ret = baytech.reboot(values[nodename], dryrun)
1214 elif continue_probe and values['model'].find("HP iLO") >= 0:
1216 hpilo = HPiLO(values, verbose, ['22'])
1217 rb_ret = hpilo.reboot(0, dryrun)
1219 hpilo = HPiLOHttps(values, verbose, ['443'])
1220 rb_ret = hpilo.reboot(0, dryrun)
1222 hpilo = HPiLOHttps(values, verbose, ['443'])
1223 rb_ret = hpilo.reboot(0, dryrun)
1226 elif continue_probe and values['model'].find("Dell RAC") >= 0:
1227 # TODO: I don't think DRACRacAdm will throw an exception for the
1228 # default method to catch...
1230 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1231 rb_ret = drac.reboot(0, dryrun)
1233 drac = DRAC(values, verbose, ['22'])
1234 rb_ret = drac.reboot(0, dryrun)
1236 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1237 wti = WTIIPS4(values, verbose, ['23'])
1238 rb_ret = wti.reboot(values[nodename], dryrun)
1240 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1241 elif continue_probe and \
1242 (values['model'].find("BlackBox PS5xx") >= 0 or
1243 values['model'].find("ePowerSwitch 1/4/8x") >=0 ):
1245 # TODO: allow a different port than http 80.
1246 if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1247 eps = ePowerSwitchGood(values, verbose, ['80'])
1248 elif values['pcu_id'] in [1003]:
1249 eps = ePowerSwitch(values, verbose, ['80'])
1251 eps = ePowerSwitchGood(values, verbose, ['80'])
1253 rb_ret = eps.reboot(values[nodename], dryrun)
1255 elif continue_probe:
1256 rb_ret = "Unsupported_PCU"
1258 elif continue_probe == False:
1259 if 'portstatus' in values:
1266 except ExceptionPort, err:
1271 #elif continue_probe and values['protocol'] == "racadm" and \
1272 # values['model'] == "RAC":
1273 # rb_ret = racadm_reboot(pcu_name(values),
1274 # values['username'],
1275 # values['password'],
1279 # Returns true if rebooted via PCU
1280 def reboot_old(nodename, dryrun):
1281 pcu = plc.getpcu(nodename)
1283 plc.nodePOD(nodename)
1286 logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1288 # APC Masterswitch (Berkeley)
1289 if pcu['model'] == "APC Masterswitch":
1290 err = apc_reboot(pcu['ip'], pcu['username'],pcu['password'],
1291 pcu[nodename], pcu['protocol'], dryrun)
1293 # DataProbe iPal (many sites)
1294 elif pcu['protocol'] == "telnet" and pcu['model'].find("IP-4") >= 0:
1295 err = ipal_reboot(pcu['ip'],pcu['password'], pcu[nodename], dryrun)
1298 elif pcu['protocol'] == "ssh" and \
1299 (pcu['model'].find("Baytech") >= 0 or pcu['model'].find("DS4") >= 0):
1300 err = baytech_reboot(pcu['ip'], pcu['username'],pcu['password'], pcu[nodename], dryrun)
1302 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1303 elif pcu['protocol'] == "http" and (pcu['model'] == "bbpse"):
1304 err = bbpse_reboot(pcu['ip'], pcu['username'], pcu['password'], pcu[nodename],80, dryrun)
1307 elif pcu['protocol'] == "ssh" and (pcu['model'] == "x10toggle"):
1308 err = x10toggle_reboot(pcu['ip'], pcu['username'],pcu['password'], pcu[nodename], dryrun)
1311 elif pcu['protocol'] == "racadm" and (pcu['model'] == "RAC"):
1312 err = racadm_reboot(pcu['ip'], pcu['username'],pcu['password'], pcu_[nodename], dryrun)
1314 # Unknown or unsupported
1316 err = errno.EPROTONOSUPPORT
1322 logger.setLevel(logging.DEBUG)
1323 ch = logging.StreamHandler()
1324 ch.setLevel(logging.DEBUG)
1325 formatter = logging.Formatter('LOGGER - %(message)s')
1326 ch.setFormatter(formatter)
1327 logger.addHandler(ch)
1330 print "Rebooting %s" % sys.argv[1]
1331 reboot_policy(sys.argv[1], True, False)
1332 except Exception, err:
1335 if __name__ == '__main__':
1337 logger = logging.getLogger("monitor")