3 # Reboot specified nodes
9 import errno, time, traceback
12 import threading, popen2
14 from monitor.wrapper import plc
16 from subprocess import PIPE, Popen
17 import pcucontrol.transports.ssh.pxssh as pxssh
18 import pcucontrol.transports.ssh.pexpect as pexpect
20 from monitor.util import command
22 # Use our versions of telnetlib and pyssh
23 sys.path.insert(0, os.path.dirname(sys.argv[0]))
24 import pcucontrol.transports.telnetlib as telnetlib
25 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")
26 import pcucontrol.transports.pyssh as pyssh
31 # Event class ID from pcu events
32 #NODE_POWER_CONTROL = 3
35 #MONITOR_USER_ID = 11142
38 logger = logging.getLogger("monitor")
42 class ExceptionNoTransport(Exception): pass
43 class ExceptionNotFound(Exception): pass
44 class ExceptionPassword(Exception): pass
45 class ExceptionTimeout(Exception): pass
46 class ExceptionPrompt(Exception): pass
47 class ExceptionSequence(Exception): pass
48 class ExceptionReset(Exception): pass
49 class ExceptionPort(Exception): pass
50 class ExceptionUsername(Exception): pass
52 def telnet_answer(telnet, expected, buffer):
55 output = telnet.read_until(expected, TELNET_TIMEOUT)
57 # logger.debug(output)
58 if output.find(expected) == -1:
59 raise ExceptionNotFound, "'%s' not found" % expected
61 telnet.write(buffer + "\r\n")
64 # PCU has model, host, preferred-port, user, passwd,
66 # This is an object derived directly form the PLCAPI DB fields
68 def __init__(self, plc_pcu_dict):
69 for field in ['username', 'password', 'site_id',
72 'node_ids', 'ports', ]:
73 if field in plc_pcu_dict:
74 self.__setattr__(field, plc_pcu_dict[field])
76 raise Exception("No such field %s in PCU object" % field)
78 # These are the convenience functions build around the PCU object.
80 def __init__(self, plc_pcu_dict):
81 PCU.__init__(self, plc_pcu_dict)
82 self.host = self.pcu_name()
85 if self.hostname is not None and self.hostname is not "":
87 elif self.ip is not None and self.ip is not "":
92 def nodeidToPort(self, node_id):
93 if node_id in self.node_ids:
94 for i in range(0, len(self.node_ids)):
95 if node_id == self.node_ids[i]:
98 raise Exception("No such Node ID: %d" % node_id)
100 # This class captures the observed pcu records from FindBadPCUs.py
102 def __init__(self, pcu_record_dict):
103 for field in ['nodenames', 'portstatus',
106 if field in pcu_record_dict:
107 if field == "reboot":
108 self.__setattr__("reboot_str", pcu_record_dict[field])
110 self.__setattr__(field, pcu_record_dict[field])
112 raise Exception("No such field %s in pcu record dict" % field)
122 def __init__(self, type, verbose):
124 self.verbose = verbose
125 self.transport = None
127 def open(self, host, username=None, password=None, prompt="User Name"):
130 if self.type == self.TELNET:
131 transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
132 transport.set_debuglevel(self.verbose)
133 if username is not None:
134 self.transport = transport
135 self.ifThenSend(prompt, username, ExceptionUsername)
137 elif self.type == self.SSH:
138 if username is not None:
139 transport = pyssh.Ssh(username, host)
140 transport.set_debuglevel(self.verbose)
142 # TODO: have an ssh set_debuglevel() also...
144 raise Exception("Username cannot be None for ssh transport.")
145 elif self.type == self.HTTP:
146 self.url = "http://%s:%d/" % (host,80)
147 uri = "%s:%d" % (host,80)
150 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
151 authinfo.add_password (None, uri, username, password)
152 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
154 transport = urllib2.build_opener(authhandler)
157 raise Exception("Unknown transport type: %s" % self.type)
159 self.transport = transport
163 if self.type == self.TELNET:
164 self.transport.close()
165 elif self.type == self.SSH:
166 self.transport.close()
167 elif self.type == self.HTTP:
170 raise Exception("Unknown transport type %s" % self.type)
171 self.transport = None
173 def sendHTTP(self, resource, data):
175 print "POSTing '%s' to %s" % (data,self.url + resource)
178 f = self.transport.open(self.url + resource ,data)
183 except urllib2.URLError,err:
184 logger.info('Could not open http connection', err)
185 return "http transport error"
189 def sendPassword(self, password, prompt=None):
190 if self.type == self.TELNET:
192 self.ifThenSend("Password", password, ExceptionPassword)
194 self.ifThenSend(prompt, password, ExceptionPassword)
195 elif self.type == self.SSH:
196 self.ifThenSend("password:", password, ExceptionPassword)
197 elif self.type == self.HTTP:
200 raise Exception("Unknown transport type: %s" % self.type)
202 def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
204 if self.transport != None:
205 output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
206 if output.find(expected) == -1:
207 raise ErrorClass, "'%s' not found" % expected
209 self.transport.write(buffer + "\r\n")
211 raise ExceptionNoTransport("transport object is type None")
213 def ifElse(self, expected, ErrorClass):
215 self.transport.read_until(expected, self.TELNET_TIMEOUT)
217 raise ErrorClass("Could not find '%s' within timeout" % expected)
220 class PCUControl(Transport,PCUModel,PCURecord):
224 def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
225 PCUModel.__init__(self, plc_pcu_record)
226 PCURecord.__init__(self, plc_pcu_record)
229 if '22' in supported_ports and self.portstatus['22'] == "open":
231 elif '23' in supported_ports and self.portstatus['23'] == "open":
232 type = Transport.TELNET
233 elif '80' in supported_ports and self.portstatus['80'] == "open":
234 type = Transport.HTTP
235 elif '443' in supported_ports and self.portstatus['443'] == "open":
236 type = Transport.HTTP
237 elif '5869' in supported_ports and self.portstatus['5869'] == "open":
238 # For DRAC cards. Racadm opens this port.
239 type = Transport.HTTP
240 elif '9100' in supported_ports and self.portstatus['9100'] == "open":
241 type = Transport.IPAL
242 elif '16992' in supported_ports and self.portstatus['16992'] == "open":
243 type = Transport.HTTP
245 raise ExceptionPort("Unsupported Port: No transport from open ports")
247 raise Exception("No Portstatus: No transport because no open ports")
248 Transport.__init__(self, type, verbose)
250 def run(self, node_port, dryrun):
251 """ This function is to be defined by the specific PCU instance. """
254 def reboot(self, node_port, dryrun):
256 return self.run(node_port, dryrun)
257 except ExceptionNotFound, err:
258 return "error: " + str(err)
259 except ExceptionPassword, err:
260 return "password exception: " + str(err)
261 except ExceptionTimeout, err:
262 return "timeout exception: " + str(err)
263 except ExceptionUsername, err:
264 return "exception: no username prompt: " + str(err)
265 except ExceptionSequence, err:
266 return "sequence error: " + str(err)
267 except ExceptionPrompt, err:
268 return "prompt exception: " + str(err)
269 except ExceptionPort, err:
270 return "no ports exception: " + str(err)
271 except socket.error, err:
272 return "socket error: timeout: " + str(err)
273 except EOFError, err:
275 logger.debug("reboot: EOF")
277 self.transport.close()
279 traceback.print_exc()
280 return "EOF connection reset" + str(err)
282 class IPAL(PCUControl):
284 This now uses a proprietary format for communicating with the PCU. I
285 prefer it to Telnet, and Web access, since it's much lighter weight
286 and, more importantly, IT WORKS!! HHAHHHAHAHAHAHAHA!
288 supported_ports = [23,80,9100]
290 def format_msg(self, data, cmd):
291 esc = chr(int('1b',16))
292 return "%c%s%c%s%c" % (esc, self.password, esc, data, cmd) # esc, 'q', chr(4))
294 def recv_noblock(self, s, count):
298 # TODO: make sleep backoff, before stopping.
300 ret = s.recv(count, socket.MSG_DONTWAIT)
301 except socket.error, e:
302 if e[0] == errno.EAGAIN:
303 raise Exception(e[1])
305 # TODO: not other exceptions.
309 def run(self, node_port, dryrun):
310 if self.type == Transport.IPAL:
311 return self.run_ipal(node_port, dryrun)
312 elif self.type == Transport.TELNET:
313 return self.run_telnet(node_port, dryrun)
315 raise Exception("Unimplemented Transport for IPAL")
317 def run_telnet(self, node_port, dryrun):
318 # TELNET version of protocol...
320 ## XXX Some iPals require you to hit Enter a few times first
321 self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
323 self.ifThenSend("Password >", self.password, ExceptionPassword)
324 self.transport.write("\r\n\r\n")
325 if not dryrun: # P# - Pulse relay
326 print "node_port %s" % node_port
327 self.ifThenSend("Enter >",
330 print "send newlines"
331 self.transport.write("\r\n\r\n")
332 print "after new lines"
333 # Get the next prompt
334 print "wait for enter"
335 self.ifElse("Enter >", ExceptionTimeout)
340 def run_ipal(self, node_port, dryrun):
346 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
349 s.connect((self.host, 9100))
350 except socket.error, e:
352 if e[0] == errno.ECONNREFUSED:
353 # cannot connect to remote host
354 raise Exception(e[1])
356 # TODO: what other conditions are there?
360 print "Checking status"
361 s.send(self.format_msg("", 'O'))
362 ret = self.recv_noblock(s, 8)
363 print "Current status is '%s'" % ret
366 raise Exception("Status returned 'another session already open' %s : %s" % (node_port, ret))
368 if node_port < len(ret):
369 status = ret[node_port]
377 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
379 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
384 print "Pulsing %s" % node_port
385 s.send(self.format_msg("%s" % node_port, 'P'))
387 # NOTE: turn power on ; do not pulse the port.
388 print "Power was off, so turning on ..."
389 s.send(self.format_msg("%s" % node_port, 'E'))
390 #s.send(self.format_msg("%s" % node_port, 'P'))
392 print "Receiving response."
393 ret = self.recv_noblock(s, 8)
394 print "Current status is '%s'" % ret
396 if node_port < len(ret):
397 status = ret[node_port]
405 raise Exception("Unknown status for PCU socket %s : %s" % (node_port, ret))
407 raise Exception("Mismatch between configured port and PCU status: %s %s" % (node_port, ret))
412 return "Failed Power On"
418 class APCEurope(PCUControl):
419 def run(self, node_port, dryrun):
420 self.open(self.host, self.username)
421 self.sendPassword(self.password)
423 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
424 self.ifThenSend("\r\n> ", "2")
425 self.ifThenSend("\r\n> ", str(node_port))
426 # 3- Immediate Reboot
427 self.ifThenSend("\r\n> ", "3")
430 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
434 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
435 "", ExceptionSequence)
436 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
441 class APCBrazil(PCUControl):
442 def run(self, node_port, dryrun):
443 self.open(self.host, self.username)
444 self.sendPassword(self.password)
446 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
447 self.ifThenSend("\r\n> ", str(node_port))
448 # 4- Immediate Reboot
449 self.ifThenSend("\r\n> ", "4")
452 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
456 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
457 "", ExceptionSequence)
458 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
463 class APCBerlin(PCUControl):
464 def run(self, node_port, dryrun):
465 self.open(self.host, self.username)
466 self.sendPassword(self.password)
468 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
469 self.ifThenSend("\r\n> ", "2")
470 self.ifThenSend("\r\n> ", "1")
471 self.ifThenSend("\r\n> ", str(node_port))
472 # 3- Immediate Reboot
473 self.ifThenSend("\r\n> ", "3")
476 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
480 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
481 "", ExceptionSequence)
482 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
487 class APCFolsom(PCUControl):
488 def run(self, node_port, dryrun):
489 self.open(self.host, self.username)
490 self.sendPassword(self.password)
492 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
493 self.ifThenSend("\r\n> ", "2")
494 self.ifThenSend("\r\n> ", "1")
495 self.ifThenSend("\r\n> ", str(node_port))
496 self.ifThenSend("\r\n> ", "1")
498 # 3- Immediate Reboot
499 self.ifThenSend("\r\n> ", "3")
502 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
506 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
507 "", ExceptionSequence)
508 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
513 class APCMaster(PCUControl):
514 supported_ports = [22,23]
515 def run(self, node_port, dryrun):
516 print "Rebooting %s" % self.host
517 self.open(self.host, self.username)
518 self.sendPassword(self.password)
521 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
522 # 3- Outlet Control/Config
523 self.ifThenSend("\r\n> ", "3")
525 self.ifThenSend("\r\n> ", str(node_port))
527 self.ifThenSend("\r\n> ", "1")
528 # 3- Immediate Reboot
529 self.ifThenSend("\r\n> ", "3")
532 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
536 self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel",
537 "", ExceptionSequence)
538 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
543 class APC(PCUControl):
544 def __init__(self, plc_pcu_record, verbose):
545 PCUControl.__init__(self, plc_pcu_record, verbose)
547 self.master = APCMaster(plc_pcu_record, verbose)
548 self.folsom = APCFolsom(plc_pcu_record, verbose)
549 self.europe = APCEurope(plc_pcu_record, verbose)
551 def run(self, node_port, dryrun):
555 for pcu in [self.master, self.europe, self.folsom]:
558 print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
561 time.sleep(sleep_time)
562 ret = pcu.reboot(node_port, dryrun)
563 except ExceptionSequence, err:
569 return "Unknown reboot sequence for APC PCU"
573 class IntelAMT(PCUControl):
574 supported_ports = [16992]
576 def run(self, node_port, dryrun):
579 # TODO: need to make this path universal; not relative to pwd.
580 cmd_str = "pcucontrol/models/intelamt/remoteControl"
583 # NOTE: -p checks the power state of the host.
584 # TODO: parse the output to find out if it's ok or not.
585 cmd_str += " -p http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
587 cmd_str += " -A http://%s:16992/RemoteControlService -user admin -pass '%s' " % (self.host, self.password )
590 return cmd.system(cmd_str, self.TELNET_TIMEOUT)
592 class DRACRacAdm(PCUControl):
593 def run(self, node_port, dryrun):
595 print "trying racadm_reboot..."
596 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
600 class DRAC(PCUControl):
601 supported_ports = [22,443,5869]
602 def run(self, node_port, dryrun):
603 self.open(self.host, self.username)
604 self.sendPassword(self.password)
606 print "logging in..."
607 self.transport.write("\r\n")
610 self.ifThenSend("[%s]#" % self.username, "getsysinfo")
613 self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
615 self.ifThenSend("[%s]#" % self.username, "exit")
620 class HPiLO(PCUControl):
621 supported_ports = [22,443]
622 def run(self, node_port, dryrun):
623 self.open(self.host, self.username)
624 self.sendPassword(self.password)
627 self.ifThenSend("</>hpiLO->", "cd system1")
629 # Reboot Outlet N (Y/N)?
631 self.ifThenSend("</system1>hpiLO->", "POWER")
634 self.ifThenSend("</system1>hpiLO->", "reset")
636 self.ifThenSend("</system1>hpiLO->", "exit")
642 class HPiLOHttps(PCUControl):
643 supported_ports = [22,443]
644 def run(self, node_port, dryrun):
646 locfg = command.CMD()
647 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
648 self.host, "iloxml/Get_Network.xml",
649 self.username, self.password)
650 sout, serr = locfg.run_noexcept(cmd)
652 if sout.strip() != "":
653 print "sout: %s" % sout.strip()
657 locfg = command.CMD()
658 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p '%s' | grep 'MESSAGE' | grep -v 'No error'" % (
659 self.host, "iloxml/Reset_Server.xml",
660 self.username, self.password)
661 sout, serr = locfg.run_noexcept(cmd)
663 if sout.strip() != "":
664 print "sout: %s" % sout.strip()
668 class BayTechAU(PCUControl):
669 def run(self, node_port, dryrun):
670 self.open(self.host, self.username, None, "Enter user name:")
671 self.sendPassword(self.password, "Enter Password:")
673 #self.ifThenSend("RPC-16>", "Status")
674 self.ifThenSend("RPC3-NC>", "Reboot %d" % node_port)
676 # Reboot Outlet N (Y/N)?
678 self.ifThenSend("(Y/N)?", "N")
680 self.ifThenSend("(Y/N)?", "Y")
681 self.ifThenSend("RPC3-NC>", "")
686 class BayTechGeorgeTown(PCUControl):
687 def run(self, node_port, dryrun):
688 self.open(self.host, self.username, None, "Enter user name:")
689 self.sendPassword(self.password, "Enter Password:")
691 #self.ifThenSend("RPC-16>", "Status")
693 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
695 # Reboot Outlet N (Y/N)?
697 self.ifThenSend("(Y/N)?", "N")
699 self.ifThenSend("(Y/N)?", "Y")
700 self.ifThenSend("RPC-16>", "")
705 class BayTechCtrlCUnibe(PCUControl):
707 For some reason, these units let you log in fine, but they hang
708 indefinitely, unless you send a Ctrl-C after the password. No idea
711 def run(self, node_port, dryrun):
712 print "BayTechCtrlC %s" % self.host
714 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
716 if not s.login(self.host, self.username, self.password, ssh_options):
717 raise ExceptionPassword("Invalid Password")
718 # Otherwise, the login succeeded.
720 # Send a ctrl-c to the remote process.
721 print "sending ctrl-c"
724 # Control Outlets (5 ,1).........5
726 #index = s.expect("Enter Request")
727 index = s.expect(["Enter Request :"])
732 index = s.expect(["DS-RPC>", "Enter user name:"])
734 s.send(self.username + "\r\n")
735 index = s.expect(["DS-RPC>"])
738 print "Reboot %d" % node_port
739 s.send("Reboot %d\r\n" % node_port)
742 index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
751 raise ExceptionPrompt("PCU Reported 'Port in use.'")
753 raise ExceptionSequence("Issued command 'Reboot' failed.")
756 index = s.expect(["DS-RPC>"])
757 #print "got prompt back"
762 raise ExceptionPrompt("EOF before expected Prompt")
763 except pexpect.TIMEOUT:
764 raise ExceptionPrompt("Timeout before expected Prompt")
768 class BayTechCtrlC(PCUControl):
770 For some reason, these units let you log in fine, but they hang
771 indefinitely, unless you send a Ctrl-C after the password. No idea
774 def run(self, node_port, dryrun):
775 print "BayTechCtrlC %s" % self.host
777 ssh_options="-o StrictHostKeyChecking=no -o PasswordAuthentication=yes -o PubkeyAuthentication=no"
779 if not s.login(self.host, self.username, self.password, ssh_options):
780 raise ExceptionPassword("Invalid Password")
781 # Otherwise, the login succeeded.
783 # Send a ctrl-c to the remote process.
784 print "SENDING ctrl-c"
787 # Control Outlets (5 ,1).........5
789 print "EXPECTING: ", "Enter Request :"
790 index = s.expect(["Enter Request :"])
795 print "EXPECTING: ", "DS-RPC>"
796 index = s.expect(["DS-RPC>", "Enter user name:", "Port in use."])
798 print "sending username"
799 s.send(self.username + "\r\n")
800 index = s.expect(["DS-RPC>"])
802 raise ExceptionPrompt("PCU Reported 'Port in use.'")
805 print "SENDING: Reboot %d" % node_port
806 s.send("Reboot %d\r\n" % node_port)
810 print "EXPECTING: ", "Y/N?"
811 index = s.expect(["\(Y/N\)\?", "Port in use", "DS-RPC>"])
820 raise ExceptionPrompt("PCU Reported 'Port in use.'")
822 raise ExceptionSequence("Issued command 'Reboot' failed.")
824 # NOTE: for some reason, the script times out with the
825 # following line. In manual tests, it works correctly, but
826 # with automated tests, evidently it fails.
829 #print "TOTAL--", s.allstr, "--EOT"
830 index = s.expect(["DS-RPC>"])
831 print "got prompt back"
836 raise ExceptionPrompt("EOF before 'Enter Request' Prompt")
837 except pexpect.TIMEOUT:
838 raise ExceptionPrompt("Timeout before Prompt")
842 class BayTech(PCUControl):
843 supported_ports = [22,23]
844 def run(self, node_port, dryrun):
845 self.open(self.host, self.username)
846 self.sendPassword(self.password)
848 # Control Outlets (5 ,1).........5
849 self.ifThenSend("Enter Request :", "5")
853 self.ifThenSend("DS-RPC>", "Reboot %d" % node_port, ExceptionNotFound)
854 except ExceptionNotFound, msg:
855 # one machine is configured to ask for a username,
856 # even after login...
857 print "msg: %s" % msg
858 self.transport.write(self.username + "\r\n")
860 self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
862 # Reboot Outlet N (Y/N)?
864 self.ifThenSend("(Y/N)?", "N")
866 self.ifThenSend("(Y/N)?", "Y")
868 self.ifThenSend("DS-RPC>", "")
873 class WTIIPS4(PCUControl):
874 supported_ports = [23]
875 def run(self, node_port, dryrun):
877 self.sendPassword(self.password, "Enter Password:")
879 self.ifThenSend("IPS> ", "/Boot %s" % node_port)
881 self.ifThenSend("Sure? (Y/N): ", "N")
883 self.ifThenSend("Sure? (Y/N): ", "Y")
885 self.ifThenSend("IPS> ", "")
890 class ePowerSwitchGood(PCUControl):
892 # The old code used Python's HTTPPasswordMgrWithDefaultRealm()
893 # For some reason this both doesn't work and in some cases, actually
894 # hangs the PCU. Definitely not what we want.
896 # The code below is much simpler. Just letting things fail first,
897 # and then, trying again with authentication string in the header.
899 def run(self, node_port, dryrun):
900 self.transport = None
901 self.url = "http://%s:%d/" % (self.host,80)
902 uri = "%s:%d" % (self.host,80)
904 req = urllib2.Request(self.url)
906 handle = urllib2.urlopen(req)
908 # NOTE: this is expected to fail initially
915 return "ERROR: not protected by HTTP authentication"
917 if not hasattr(e, 'code') or e.code != 401:
918 return "ERROR: failed for: %s" % str(e)
920 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
921 # NOTE: assuming basic realm authentication.
922 authheader = "Basic %s" % base64data
923 req.add_header("Authorization", authheader)
926 f = urllib2.urlopen(req)
928 # failing here means the User/passwd is wrong (hopefully)
929 raise ExceptionPassword("Incorrect username/password")
931 # NOTE: after verifying that the user/password is correct,
932 # actually reboot the given node.
935 data = urllib.urlencode({'P%d' % node_port : "r"})
936 req = urllib2.Request(self.url + "cmd.html")
937 req.add_header("Authorization", authheader)
938 # add data to handler,
939 f = urllib2.urlopen(req, data)
940 if self.verbose: print f.read()
942 import traceback; traceback.print_exc()
944 # fetch url one more time on cmd.html, econtrol.html or whatever.
947 if self.verbose: print f.read()
952 class CustomPCU(PCUControl):
953 def run(self, node_port, dryrun):
954 url = "https://www-itec.uni-klu.ac.at/plab-pcu/index.php"
957 # Turn host off, then on
958 formstr = "plab%s=off" % node_port
959 os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
961 formstr = "plab%s=on" % node_port
962 os.system("curl --user %s:%s --form '%s' --insecure %s" % (self.username, self.password, formstr, url))
964 os.system("curl --user %s:%s --insecure %s" % (self.username, self.password, url))
967 class ePowerSwitchOld(PCUControl):
968 def run(self, node_port, dryrun):
969 self.url = "http://%s:%d/" % (self.host,80)
970 uri = "%s:%d" % (self.host,80)
973 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
974 authinfo.add_password (None, uri, self.username, self.password)
975 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
977 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
978 transport = urllib2.build_opener(authinfo)
979 f = transport.open(self.url)
980 if self.verbose: print f.read()
983 transport = urllib2.build_opener(authhandler)
984 f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
985 if self.verbose: print f.read()
990 class ePowerSwitch(PCUControl):
991 supported_ports = [80]
992 def run(self, node_port, dryrun):
993 self.url = "http://%s:%d/" % (self.host,80)
994 uri = "%s:%d" % (self.host,80)
996 # TODO: I'm still not sure what the deal is here.
997 # two independent calls appear to need to be made before the
998 # reboot will succeed. It doesn't seem to be possible to do
999 # this with a single call. I have no idea why.
1002 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1003 authinfo.add_password (None, uri, self.username, self.password)
1004 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1006 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
1007 transport = urllib2.build_opener()
1008 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
1009 if self.verbose: print f.read()
1012 transport = urllib2.build_opener(authhandler)
1013 f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
1014 if self.verbose: print f.read()
1016 # data= "P%d=r" % node_port
1017 #self.open(self.host, self.username, self.password)
1018 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
1019 #self.sendHTTP("econtrol.html", data)
1020 #self.sendHTTP("cmd.html", data)
1026 ### rebooting european BlackBox PSE boxes
1027 # Thierry Parmentelat - May 11 2005
1028 # tested on 4-ports models known as PSE505-FR
1029 # uses http to POST a data 'P<port>=r'
1030 # relies on basic authentication within http1.0
1031 # first curl-based script was
1032 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
1033 # http://<hostname>:<http_port>/cmd.html && echo OK
1035 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
1039 url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
1040 data= "P%d=r" % port_in_pcu
1042 logger.debug("POSTing '%s' on %s" % (data,url))
1044 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
1045 uri = "%s:%d" % (pcu_ip,http_port)
1046 authinfo.add_password (None, uri, username, password)
1047 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
1049 opener = urllib2.build_opener(authhandler)
1050 urllib2.install_opener(opener)
1056 f = urllib2.urlopen(url,data)
1063 except urllib2.URLError,err:
1064 logger.info('Could not open http connection', err)
1065 return "bbpse error"
1067 ### rebooting x10toggle based systems addressed by port
1068 # Marc E. Fiuczynski - May 31 2005
1069 # tested on 4-ports models known as PSE505-FR
1070 # uses ssh and password to login to an account
1071 # that will cause the system to be powercycled.
1073 def x10toggle_reboot(ip, username, password, port, dryrun):
1078 ssh = pyssh.Ssh(username, ip)
1082 telnet_answer(ssh, "password:", password)
1086 telnet_answer(ssh, "x10toggle>", "A%d" % port)
1089 output = ssh.close()
1091 logger.debug(output)
1094 except Exception, err:
1098 output = ssh.close()
1100 logger.debug(output)
1101 return errno.ETIMEDOUT
1103 ### rebooting Dell systems via RAC card
1104 # Marc E. Fiuczynski - June 01 2005
1105 # tested with David Lowenthal's itchy/scratchy nodes at UGA
1108 def runcmd(command, args, username, password, timeout = None):
1111 result_ready = threading.Condition()
1115 result_ready.acquire()
1119 result_ready.notify()
1120 result_ready.release()
1122 def do_command(command, username, password):
1125 # Popen4 is a popen-type class that combines stdout and stderr
1126 p = popen2.Popen4(command)
1128 # read all output data
1129 p.tochild.write("%s\n" % username)
1130 p.tochild.write("%s\n" % password)
1132 data = p.fromchild.read()
1135 # might get interrupted by a signal in poll() or waitpid()
1138 set_result((retval, data))
1141 if ex.errno == errno.EINTR:
1144 except Exception, ex:
1148 command = " ".join([command] + args)
1150 worker = threading.Thread(target = do_command, args = (command, username, password, ))
1151 worker.setDaemon(True)
1152 result_ready.acquire()
1154 result_ready.wait(timeout)
1156 if result == [None]:
1157 raise Exception, "command timed-out: '%s'" % command
1159 result_ready.release()
1162 if isinstance(result, Exception):
1165 (retval, data) = result
1166 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
1169 out = "system command ('%s') " % command
1170 if os.WIFEXITED(retval):
1171 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
1173 out += "killed by signal %d" % os.WTERMSIG(retval)
1175 out += "; output follows:\n" + data
1176 raise Exception, out
1178 def racadm_reboot(host, username, password, port, dryrun):
1181 ip = socket.gethostbyname(host)
1183 cmd = "/usr/sbin/racadm"
1186 output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
1189 output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
1192 print "RUNCMD: %s" % output
1194 logger.debug(output)
1197 except Exception, err:
1198 logger.debug("runcmd raised exception %s" % err)
1204 if pcu['hostname'] is not None and pcu['hostname'] is not "":
1205 return pcu['hostname']
1206 elif pcu['ip'] is not None and pcu['ip'] is not "":
1212 from monitor import database
1215 def get_pcu_values(pcu_id):
1218 # this shouldn't be loaded each time...
1219 fb = database.dbLoad("findbadpcus")
1222 values = fb['nodes']["id_%s" % pcu_id]['values']
1228 def reboot(nodename):
1229 return reboot_policy(nodename, True, False)
1231 def reboot_policy(nodename, continue_probe, dryrun):
1234 pcu = plc.getpcu(nodename)
1236 logger.debug("no pcu for %s" % hostname)
1237 print "no pcu for %s" % hostname
1238 return False # "%s has no pcu" % nodename
1240 values = get_pcu_values(pcu['pcu_id'])
1242 logger.debug("No values for pcu probe %s" % hostname)
1243 print "No values for pcu probe %s" % hostname
1244 return False #"no info for pcu_id %s" % pcu['pcu_id']
1247 logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1249 ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
1258 class Unknown(PCUControl):
1259 supported_ports = [22,23,80,443,5869,9100,16992]
1261 def model_to_object(modelname):
1262 if "AMT" in modelname:
1264 elif "DS4-RPC" in modelname:
1266 elif "ilo2" in modelname:
1268 elif "IP-41x" in modelname:
1270 elif "AP79xx" in modelname or "Masterswitch" in modelname:
1272 elif "DRAC" in modelname:
1274 elif "WTI" in modelname:
1276 elif "ePowerSwitch" in modelname:
1281 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
1283 if 'plc_pcu_stats' in values:
1284 values.update(values['plc_pcu_stats'])
1287 # DataProbe iPal (many sites)
1288 if continue_probe and values['model'].find("IP-41x_IP-81x") >= 0:
1289 ipal = IPAL(values, verbose, ['23', '80', '9100'])
1290 rb_ret = ipal.reboot(values[nodename], dryrun)
1292 # APC Masterswitch (Berkeley)
1293 elif continue_probe and ( values['model'].find("AP79xx") >= 0 or \
1294 values['model'].find("Masterswitch") >= 0 ):
1297 # TODO: make a more robust version of APC
1298 if values['pcu_id'] in [1102,1163,1055,1111,1231,1113,1127,1128,1148]:
1299 apc = APCEurope(values, verbose, ['22', '23'])
1300 rb_ret = apc.reboot(values[nodename], dryrun)
1302 elif values['pcu_id'] in [1110,86]:
1303 apc = APCBrazil(values, verbose, ['22', '23'])
1304 rb_ret = apc.reboot(values[nodename], dryrun)
1306 elif values['pcu_id'] in [1221,1225,1220]:
1307 apc = APCBerlin(values, verbose, ['22', '23'])
1308 rb_ret = apc.reboot(values[nodename], dryrun)
1310 elif values['pcu_id'] in [1173,1240,47]:
1311 apc = APCFolsom(values, verbose, ['22', '23'])
1312 rb_ret = apc.reboot(values[nodename], dryrun)
1315 apc = APCMaster(values, verbose, ['22', '23'])
1316 rb_ret = apc.reboot(values[nodename], dryrun)
1319 elif continue_probe and values['model'].find("DS4-RPC") >= 0:
1320 if values['pcu_id'] in [1056,1237,1052,1209,1002,1008,1041,1013,1022]:
1321 # These require a 'ctrl-c' to be sent...
1322 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1323 rb_ret = baytech.reboot(values[nodename], dryrun)
1325 elif values['pcu_id'] in [93]:
1326 baytech = BayTechAU(values, verbose, ['22', '23'])
1327 rb_ret = baytech.reboot(values[nodename], dryrun)
1329 elif values['pcu_id'] in [1057]:
1330 # These require a 'ctrl-c' to be sent...
1331 baytech = BayTechCtrlCUnibe(values, verbose, ['22', '23'])
1332 rb_ret = baytech.reboot(values[nodename], dryrun)
1334 elif values['pcu_id'] in [1012]:
1335 # This pcu sometimes doesn't present the 'Username' prompt,
1336 # unless you immediately try again...
1338 baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1339 rb_ret = baytech.reboot(values[nodename], dryrun)
1341 baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1342 rb_ret = baytech.reboot(values[nodename], dryrun)
1344 baytech = BayTech(values, verbose, ['22', '23'])
1345 rb_ret = baytech.reboot(values[nodename], dryrun)
1348 elif continue_probe and values['model'].find("ilo") >= 0:
1350 hpilo = HPiLO(values, verbose, ['22'])
1351 rb_ret = hpilo.reboot(0, dryrun)
1353 hpilo = HPiLOHttps(values, verbose, ['443'])
1354 rb_ret = hpilo.reboot(0, dryrun)
1356 hpilo = HPiLOHttps(values, verbose, ['443'])
1357 rb_ret = hpilo.reboot(0, dryrun)
1360 elif continue_probe and values['model'].find("DRAC") >= 0:
1361 # TODO: I don't think DRACRacAdm will throw an exception for the
1362 # default method to catch...
1364 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1365 rb_ret = drac.reboot(0, dryrun)
1367 drac = DRAC(values, verbose, ['22'])
1368 rb_ret = drac.reboot(0, dryrun)
1370 elif continue_probe and values['model'].find("WTI IPS-4") >= 0:
1371 wti = WTIIPS4(values, verbose, ['23'])
1372 rb_ret = wti.reboot(values[nodename], dryrun)
1374 elif continue_probe and values['model'].find("AMT") >= 0:
1375 amt = IntelAMT(values, verbose, ['16992'])
1376 rb_ret = amt.reboot(values[nodename], dryrun)
1378 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1379 elif continue_probe and values['model'].find("ePowerSwitch") >=0:
1380 # TODO: allow a different port than http 80.
1381 if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1382 eps = ePowerSwitchGood(values, verbose, ['80'])
1383 elif values['pcu_id'] in [1003]:
1386 eps = ePowerSwitch(values, verbose, ['80'])
1388 eps = ePowerSwitchGood(values, verbose, ['80'])
1390 rb_ret = eps.reboot(values[nodename], dryrun)
1391 elif continue_probe and values['pcu_id'] in [1122]:
1392 custom = CustomPCU(values, verbose, ['80', '443'])
1393 custom.reboot(values[nodename], dryrun)
1395 elif continue_probe:
1396 rb_ret = "Unsupported_PCU"
1398 elif continue_probe == False:
1399 if 'portstatus' in values:
1406 except ExceptionPort, err:
1411 #elif continue_probe and values['protocol'] == "racadm" and \
1412 # values['model'] == "RAC":
1413 # rb_ret = racadm_reboot(pcu_name(values),
1414 # values['username'],
1415 # values['password'],
1420 logger.setLevel(logging.DEBUG)
1421 ch = logging.StreamHandler()
1422 ch.setLevel(logging.DEBUG)
1423 formatter = logging.Formatter('LOGGER - %(message)s')
1424 ch.setFormatter(formatter)
1425 logger.addHandler(ch)
1428 if "test" in sys.argv:
1433 for node in sys.argv[1:]:
1434 if node == "test": continue
1436 print "Rebooting %s" % node
1437 if reboot_policy(node, True, dryrun):
1441 except Exception, err:
1442 import traceback; traceback.print_exc()
1445 if __name__ == '__main__':
1447 logger = logging.getLogger("monitor")