Better code for PCU types. Class based with specific exceptions for different
[monitor.git] / reboot.py
1 #!/usr/bin/python
2 #
3 # Reboot specified nodes
4 #
5
6 import getpass, getopt
7 import os, sys
8 import xml, xmlrpclib
9 import errno, time, traceback
10 import urllib2
11 import threading, popen2
12 import array, struct
13 #from socket import *
14 import socket
15 import plc
16 import base64
17 from subprocess import PIPE, Popen
18
19 plc_lock = threading.Lock()
20
21 # Use our versions of telnetlib and pyssh
22 sys.path.insert(0, os.path.dirname(sys.argv[0]))
23 import telnetlib
24 sys.path.insert(0, os.path.dirname(sys.argv[0]) + "/pyssh")    
25 import pyssh
26
27 # Timeouts in seconds
28 TELNET_TIMEOUT = 30
29
30 # Event class ID from pcu events
31 #NODE_POWER_CONTROL = 3
32
33 # Monitor user ID
34 #MONITOR_USER_ID = 11142
35
36 import logging
37 logger = logging.getLogger("monitor")
38 verbose = 1
39 #dryrun = 0;
40
41 class ExceptionNoTransport(Exception): pass
42 class ExceptionNotFound(Exception): pass
43 class ExceptionPassword(Exception): pass
44 class ExceptionTimeout(Exception): pass
45 class ExceptionPrompt(Exception): pass
46 class ExceptionSequence(Exception): pass
47 class ExceptionReset(Exception): pass
48 class ExceptionPort(Exception): pass
49 class ExceptionUsername(Exception): pass
50
51 def telnet_answer(telnet, expected, buffer):
52         global verbose
53
54         output = telnet.read_until(expected, TELNET_TIMEOUT)
55         #if verbose:
56         #       logger.debug(output)
57         if output.find(expected) == -1:
58                 raise ExceptionNotFound, "'%s' not found" % expected
59         else:
60                 telnet.write(buffer + "\r\n")
61
62
63 # PCU has model, host, preferred-port, user, passwd, 
64
65 # This is an object derived directly form the PLCAPI DB fields
66 class PCU(object):
67         def __init__(self, plc_pcu_dict):
68                 for field in ['username', 'password', 'site_id', 
69                                                 'hostname', 'ip', 
70                                                 'pcu_id', 'model', 
71                                                 'node_ids', 'ports', ]:
72                         if field in plc_pcu_dict:
73                                 self.__setattr__(field, plc_pcu_dict[field])
74                         else:
75                                 raise Exception("No such field %s in PCU object" % field)
76
77 # These are the convenience functions build around the PCU object.
78 class PCUModel(PCU):
79         def __init__(self, plc_pcu_dict):
80                 PCU.__init__(self, plc_pcu_dict)
81                 self.host = self.pcu_name()
82
83         def pcu_name(self):
84                 if self.hostname is not None and self.hostname is not "":
85                         return self.hostname
86                 elif self.ip is not None and self.ip is not "":
87                         return self.ip
88                 else:
89                         return None
90
91         def nodeidToPort(self, node_id):
92                 if node_id in self.node_ids:
93                         for i in range(0, len(self.node_ids)):
94                                 if node_id == self.node_ids[i]:
95                                         return self.ports[i]
96
97                 raise Exception("No such Node ID: %d" % node_id)
98
99 # This class captures the observed pcu records from FindBadPCUs.py
100 class PCURecord:
101         def __init__(self, pcu_record_dict):
102                 for field in ['nodenames', 'portstatus', 
103                                                 'dnsmatch', 
104                                                 'complete_entry', ]:
105                         if field in pcu_record_dict:
106                                 if field == "reboot":
107                                         self.__setattr__("reboot_str", pcu_record_dict[field])
108                                 else:
109                                         self.__setattr__(field, pcu_record_dict[field])
110                         else:
111                                 raise Exception("No such field %s in pcu record dict" % field)
112
113 class Transport:
114         TELNET = 1
115         SSH    = 2
116         HTTP   = 3
117
118         TELNET_TIMEOUT = 60
119
120         def __init__(self, type, verbose):
121                 self.type = type
122                 self.verbose = verbose
123                 self.transport = None
124
125 #       def __del__(self):
126 #               if self.transport:
127 #                       self.close()
128
129         def open(self, host, username=None, password=None, prompt="User Name"):
130                 transport = None
131
132                 if self.type == self.TELNET:
133                         transport = telnetlib.Telnet(host, timeout=self.TELNET_TIMEOUT)
134                         transport.set_debuglevel(self.verbose)
135                         if username is not None:
136                                 self.transport = transport
137                                 self.ifThenSend(prompt, username, ExceptionUsername)
138
139                 elif self.type == self.SSH:
140                         if username is not None:
141                                 transport = pyssh.Ssh(username, host)
142                                 transport.set_debuglevel(self.verbose)
143                                 transport.open()
144                                 # TODO: have an ssh set_debuglevel() also...
145                         else:
146                                 raise Exception("Username cannot be None for ssh transport.")
147                 elif self.type == self.HTTP:
148                         self.url = "http://%s:%d/" % (host,80)
149                         uri = "%s:%d" % (host,80)
150
151                         # create authinfo
152                         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
153                         authinfo.add_password (None, uri, username, password)
154                         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
155
156                         transport = urllib2.build_opener(authhandler)
157
158                 else:
159                         raise Exception("Unknown transport type: %s" % self.type)
160
161                 self.transport = transport
162                 return True
163
164         def close(self):
165                 if self.type == self.TELNET:
166                         self.transport.close() 
167                 elif self.type == self.SSH:
168                         self.transport.close() 
169                 elif self.type == self.HTTP:
170                         pass
171                 else:
172                         raise Exception("Unknown transport type %s" % self.type)
173                 self.transport = None
174
175         def sendHTTP(self, resource, data):
176                 if self.verbose:
177                         print "POSTing '%s' to %s" % (data,self.url + resource)
178
179                 try:
180                         f = self.transport.open(self.url + resource ,data)
181                         r = f.read()
182                         if self.verbose:
183                                 print r
184
185                 except urllib2.URLError,err:
186                         logger.info('Could not open http connection', err)
187                         return "http transport error"
188
189                 return 0
190
191         def sendPassword(self, password, prompt=None):
192                 if self.type == self.TELNET:
193                         if prompt == None:
194                                 self.ifThenSend("Password", password, ExceptionPassword)
195                         else:
196                                 self.ifThenSend(prompt, password, ExceptionPassword)
197                 elif self.type == self.SSH:
198                         self.ifThenSend("password:", password, ExceptionPassword)
199                 elif self.type == self.HTTP:
200                         pass
201                 else:
202                         raise Exception("Unknown transport type: %s" % self.type)
203
204         def ifThenSend(self, expected, buffer, ErrorClass=ExceptionPrompt):
205
206                 if self.transport != None:
207                         output = self.transport.read_until(expected, self.TELNET_TIMEOUT)
208                         if output.find(expected) == -1:
209                                 raise ErrorClass, "'%s' not found" % expected
210                         else:
211                                 self.transport.write(buffer + "\r\n")
212                 else:
213                         raise ExceptionNoTransport("transport object is type None")
214
215         def ifElse(self, expected, ErrorClass):
216                 try:
217                         self.transport.read_until(expected, self.TELNET_TIMEOUT)
218                 except:
219                         raise ErrorClass("Could not find '%s' within timeout" % expected)
220                         
221
222 class PCUControl(Transport,PCUModel,PCURecord):
223         def __init__(self, plc_pcu_record, verbose, supported_ports=[]):
224                 PCUModel.__init__(self, plc_pcu_record)
225                 PCURecord.__init__(self, plc_pcu_record)
226                 if self.portstatus:
227                         if '22' in supported_ports and self.portstatus['22'] == "open":
228                                 type = Transport.SSH
229                         elif '23' in supported_ports and self.portstatus['23'] == "open":
230                                 type = Transport.TELNET
231                         elif '80' in supported_ports and self.portstatus['80'] == "open":
232                                 type = Transport.HTTP
233                         elif '443' in supported_ports and self.portstatus['443'] == "open":
234                                 type = Transport.HTTP
235                         elif '5869' in supported_ports and self.portstatus['5869'] == "open":
236                                 # For DRAC cards.  not sure how much it's used in the
237                                 # protocol.. but racadm opens this port.
238                                 type = Transport.HTTP
239                         else:
240                                 raise ExceptionPort("Unsupported Port: No transport from open ports")
241                 Transport.__init__(self, type, verbose)
242
243         def run(self, node_port, dryrun):
244                 """ This function is to be defined by the specific PCU instance.  """
245                 pass
246                 
247         def reboot(self, node_port, dryrun):
248                 try:
249                         return self.run(node_port, dryrun)
250                 except ExceptionNotFound, err:
251                         return "error: " + str(err)
252                 except ExceptionPassword, err:
253                         return "password exception: " + str(err)
254                 except ExceptionTimeout, err:
255                         return "timeout exception: " + str(err)
256                 except ExceptionUsername, err:
257                         return "exception: no username prompt: " + str(err)
258                 except ExceptionSequence, err:
259                         return "sequence error: " + str(err)
260                 except ExceptionPrompt, err:
261                         return "prompt exception: " + str(err)
262                 except ExceptionPort, err:
263                         return "no ports exception: " + str(err)
264                 except socket.error, err:
265                         return "socket error: timeout: " + str(err)
266                 except EOFError, err:
267                         if self.verbose:
268                                 logger.debug("reboot: EOF")
269                                 logger.debug(err)
270                         self.transport.close()
271                         import traceback
272                         traceback.print_exc()
273                         return "EOF connection reset" + str(err)
274                 #except Exception, err:
275                 #       if self.verbose:
276                 #               logger.debug("reboot: Exception")
277                 #               logger.debug(err)
278                 #       if self.transport:
279                 #               self.transport.close()
280                 #       import traceback
281                 #       traceback.print_exc()
282                 #       return  "generic exception; unknown problem."
283
284                 
285 class IPAL(PCUControl):
286         def run(self, node_port, dryrun):
287                 self.open(self.host)
288
289                 # XXX Some iPals require you to hit Enter a few times first
290                 self.ifThenSend("Password >", "\r\n\r\n", ExceptionNotFound)
291
292                 # Login
293                 self.ifThenSend("Password >", self.password, ExceptionPassword)
294                 self.transport.write("\r\n\r\n")
295
296                 if not dryrun: # P# - Pulse relay
297                         self.ifThenSend("Enter >", 
298                                                         "P%d" % node_port, 
299                                                         ExceptionNotFound)
300                 # Get the next prompt
301                 self.ifElse("Enter >", ExceptionTimeout)
302
303                 self.close()
304                 return 0
305
306 def ipal_reboot(ip, password, port, dryrun):
307         global verbose
308         global plc_lock
309         telnet = None
310
311         try:
312                 #plc_lock.acquire()
313                 #print "lock acquired"
314
315                 #try:
316                         #telnet = telnetlib.Telnet(ip) # , timeout=TELNET_TIMEOUT)
317                 telnet = telnetlib.Telnet(ip, timeout=TELNET_TIMEOUT)
318                 #except:
319                 #       import traceback
320                 #       traceback.print_exc()
321
322
323                 telnet.set_debuglevel(verbose)
324
325                 # XXX Some iPals require you to hit Enter a few times first
326                 telnet_answer(telnet, "Password >", "\r\n\r\n")
327
328                 # Login
329                 telnet_answer(telnet, "Password >", password)
330
331                 # XXX Some iPals require you to hit Enter a few times first
332                 telnet.write("\r\n\r\n")
333
334                 # P# - Pulse relay
335                 if not dryrun:
336                         telnet_answer(telnet, "Enter >", "P%d" % port)
337
338                 telnet.read_until("Enter >", TELNET_TIMEOUT)
339
340                 # Close
341                 telnet.close()
342
343                 #print "lock released"
344                 #plc_lock.release()
345                 return 0
346
347         except EOFError, err:
348                 if verbose:
349                         logger.debug("ipal_reboot: EOF")
350                         logger.debug(err)
351                 telnet.close()
352                 import traceback
353                 traceback.print_exc()
354                 #print "lock released"
355                 #plc_lock.release()
356                 return errno.ECONNRESET
357         except socket.error, err:
358                 logger.debug("ipal_reboot: Socket Error")
359                 logger.debug(err)
360                 import traceback
361                 traceback.print_exc()
362
363                 return errno.ETIMEDOUT
364                 
365         except Exception, err:
366                 if verbose:
367                         logger.debug("ipal_reboot: Exception")
368                         logger.debug(err)
369                 if telnet:
370                         telnet.close()
371                 import traceback
372                 traceback.print_exc()
373                 #print "lock released"
374                 #plc_lock.release()
375                 return  "ipal error"
376
377 class APCEurope(PCUControl):
378         def run(self, node_port, dryrun):
379                 self.open(self.host, self.username)
380                 self.sendPassword(self.password)
381
382                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
383                 self.ifThenSend("\r\n> ", "2")
384                 self.ifThenSend("\r\n> ", str(node_port))
385                 # 3- Immediate Reboot             
386                 self.ifThenSend("\r\n> ", "3")
387
388                 if not dryrun:
389                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel", 
390                                                         "YES\r\n",
391                                                         ExceptionSequence)
392                 else:
393                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel", 
394                                                         "", ExceptionSequence)
395                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
396
397                 self.close()
398                 return 0
399
400 class APCFolsom(PCUControl):
401         def run(self, node_port, dryrun):
402                 self.open(self.host, self.username)
403                 self.sendPassword(self.password)
404
405                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
406                 self.ifThenSend("\r\n> ", "2")
407                 self.ifThenSend("\r\n> ", "1")
408                 self.ifThenSend("\r\n> ", str(node_port))
409                 self.ifThenSend("\r\n> ", "1")
410
411                 # 3- Immediate Reboot             
412                 self.ifThenSend("\r\n> ", "3")
413
414                 if not dryrun:
415                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel", 
416                                                         "YES\r\n",
417                                                         ExceptionSequence)
418                 else:
419                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel", 
420                                                         "", ExceptionSequence)
421                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
422
423                 self.close()
424                 return 0
425
426 class APCMaster(PCUControl):
427         def run(self, node_port, dryrun):
428                 self.open(self.host, self.username)
429                 self.sendPassword(self.password)
430
431                 # 1- Device Manager
432                 self.ifThenSend("\r\n> ", "1", ExceptionPassword)
433                 # 3- Outlet Control/Config
434                 self.ifThenSend("\r\n> ", "3")
435                 # n- Outlet n
436                 self.ifThenSend("\r\n> ", str(node_port))
437                 # 1- Control Outlet
438                 self.ifThenSend("\r\n> ", "1")
439                 # 3- Immediate Reboot             
440                 self.ifThenSend("\r\n> ", "3")
441
442                 if not dryrun:
443                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel", 
444                                                         "YES\r\n",
445                                                         ExceptionSequence)
446                 else:
447                         self.ifThenSend("Enter 'YES' to continue or <ENTER> to cancel", 
448                                                         "", ExceptionSequence)
449                 self.ifThenSend("Press <ENTER> to continue...", "", ExceptionSequence)
450
451                 self.close()
452                 return 0
453
454 class APC(PCUControl):
455         def __init__(self, plc_pcu_record, verbose):
456                 PCUControl.__init__(self, plc_pcu_record, verbose)
457
458                 self.master = APCMaster(plc_pcu_record, verbose)
459                 self.folsom = APCFolsom(plc_pcu_record, verbose)
460                 self.europe = APCEurope(plc_pcu_record, verbose)
461
462         def run(self, node_port, dryrun):
463                 try_again = True
464                 sleep_time = 1
465
466                 for pcu in [self.master, self.europe, self.folsom]:
467                         if try_again:
468                                 try:
469                                         print "-*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*"
470                                         try_again = False
471                                         print "sleeping 5"
472                                         time.sleep(sleep_time)
473                                         ret = pcu.reboot(node_port, dryrun)
474                                 except ExceptionSequence, err:
475                                         del pcu
476                                         sleep_time = 130
477                                         try_again = True
478
479                 if try_again:
480                         return "Unknown reboot sequence for APC PCU"
481                 else:
482                         return ret
483
484 class DRACRacAdm(PCUControl):
485         def run(self, node_port, dryrun):
486
487                 print "trying racadm_reboot..."
488                 racadm_reboot(self.host, self.username, self.password, node_port, dryrun)
489
490                 return 0
491
492 class DRAC(PCUControl):
493         def run(self, node_port, dryrun):
494                 self.open(self.host, self.username)
495                 self.sendPassword(self.password)
496
497                 print "logging in..."
498                 self.transport.write("\r\n")
499                 # Testing Reboot ?
500                 if dryrun:
501                         self.ifThenSend("[%s]#" % self.username, "getsysinfo")
502                 else:
503                         # Reset this machine
504                         self.ifThenSend("[%s]#" % self.username, "serveraction powercycle")
505
506                 self.ifThenSend("[%s]#" % self.username, "exit")
507
508                 self.close()
509                 return 0
510
511 class HPiLO(PCUControl):
512         def run(self, node_port, dryrun):
513                 self.open(self.host, self.username)
514                 self.sendPassword(self.password)
515
516                 # </>hpiLO-> 
517                 self.ifThenSend("</>hpiLO->", "cd system1")
518
519                 # Reboot Outlet  N        (Y/N)?
520                 if dryrun:
521                         self.ifThenSend("</system1>hpiLO->", "POWER")
522                 else:
523                         # Reset this machine
524                         self.ifThenSend("</system1>hpiLO->", "reset")
525
526                 self.ifThenSend("</system1>hpiLO->", "exit")
527
528                 self.close()
529                 return 0
530
531                 
532 class HPiLOHttps(PCUControl):
533         def run(self, node_port, dryrun):
534
535                 cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p %s" % (
536                                         self.host, "iloxml/Get_Network.xml", 
537                                         self.username, self.password)
538                 p_ilo  = Popen(cmd, stdout=PIPE, shell=True)
539                 cmd2 = "grep 'MESSAGE' | grep -v 'No error'"
540                 p_grep = Popen(cmd2, stdin=p_ilo.stdout, stdout=PIPE, stderr=PIPE, shell=True)
541                 sout, serr = p_grep.communicate()
542
543                 p_ilo.wait()
544                 p_grep.wait()
545                 if sout.strip() != "":
546                         print "sout: %s" % sout.strip()
547                         return sout.strip()
548
549                 if not dryrun:
550                         cmd = "cmdhttps/locfg.pl -s %s -f %s -u %s -p %s" % (
551                                         self.host, "iloxml/Reset_Server.xml", 
552                                         self.username, self.password)
553                         p_ilo = Popen(cmd, stdin=PIPE, stdout=PIPE, shell=True)
554                         cmd2 = "grep 'MESSAGE' | grep -v 'No error'"
555                         p_grep = Popen(cmd2, stdin=p_ilo.stdout, stdout=PIPE, stderr=PIPE)
556                         sout, serr = p_grep.communicate()
557                         p_ilo.wait()
558                         p_grep.wait()
559
560                         if sout.strip() != "":
561                                 print "sout: %s" % sout.strip()
562                                 return sout.strip()
563
564                 return 0
565
566 class BayTechGeorgeTown(PCUControl):
567         def run(self, node_port, dryrun):
568                 self.open(self.host, self.username, None, "Enter user name:")
569                 self.sendPassword(self.password, "Enter Password:")
570
571                 #self.ifThenSend("RPC-16>", "Status")
572
573                 self.ifThenSend("RPC-16>", "Reboot %d" % node_port)
574
575                 # Reboot Outlet  N        (Y/N)?
576                 if dryrun:
577                         self.ifThenSend("(Y/N)?", "N")
578                 else:
579                         self.ifThenSend("(Y/N)?", "Y")
580                 self.ifThenSend("RPC-16>", "")
581
582                 self.close()
583                 return 0
584
585 class BayTechCtrlC(PCUControl):
586         """
587                 For some reason, these units let you log in fine, but they hang
588                 indefinitely, unless you send a Ctrl-C after the password.  No idea
589                 why.
590         """
591         def run(self, node_port, dryrun):
592                 print "BayTechCtrlC %s" % self.host
593                 self.open(self.host, self.username)
594                 self.sendPassword(self.password)
595
596                 #self.transport.write('\ 3')
597                 self.transport.write("\r\n")
598                 self.transport.write(pyssh.CTRL_C)
599                 #self.transport.write(chr(3))
600                 #self.transport.write(chr(24))
601                 #self.transport.write(chr(26))
602                 #self.transport.write('\18')
603                 # Control Outlets  (5 ,1).........5
604                 self.ifThenSend("Enter Request :", "5")
605
606                 # Reboot N
607                 try:
608                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
609                 except ExceptionNotFound, msg:
610                         # one machine is configured to ask for a username,
611                         # even after login...
612                         print "msg: %s" % msg
613                         self.transport.write(self.username + "\r\n")
614                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
615                         
616
617                 # Reboot Outlet  N        (Y/N)?
618                 if dryrun:
619                         self.ifThenSend("(Y/N)?", "N")
620                 else:
621                         self.ifThenSend("(Y/N)?", "Y")
622                 self.ifThenSend("DS-RPC>", "")
623
624                 self.close()
625                 return 0
626
627 class BayTech(PCUControl):
628         def run(self, node_port, dryrun):
629                 self.open(self.host, self.username)
630                 self.sendPassword(self.password)
631
632                 # Control Outlets  (5 ,1).........5
633                 self.ifThenSend("Enter Request :", "5")
634
635                 # Reboot N
636                 try:
637                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
638                 except ExceptionNotFound, msg:
639                         # one machine is configured to ask for a username,
640                         # even after login...
641                         print "msg: %s" % msg
642                         self.transport.write(self.username + "\r\n")
643                         self.ifThenSend("DS-RPC>", "Reboot %d" % node_port)
644                         
645
646                 # Reboot Outlet  N        (Y/N)?
647                 if dryrun:
648                         self.ifThenSend("(Y/N)?", "N")
649                 else:
650                         self.ifThenSend("(Y/N)?", "Y")
651                 self.ifThenSend("DS-RPC>", "")
652
653                 self.close()
654                 return 0
655
656 class ePowerSwitchGood(PCUControl):
657         # NOTE:
658         #               The old code used Python's HTTPPasswordMgrWithDefaultRealm()
659         #               For some reason this both doesn't work and in some cases, actually
660         #               hangs the PCU.  Definitely not what we want.
661         #               
662         #               The code below is much simpler.  Just letting things fail first,
663         #               and then, trying again with authentication string in the header.
664         #               
665         def run(self, node_port, dryrun):
666                 self.transport = None
667                 self.url = "http://%s:%d/" % (self.host,80)
668                 uri = "%s:%d" % (self.host,80)
669
670                 req = urllib2.Request(self.url)
671                 try:
672                         handle = urllib2.urlopen(req)
673                 except IOError, e:
674                         # NOTE: this is expected to fail initially
675                         pass
676                 else:
677                         print self.url
678                         print "-----------"
679                         print handle.read()
680                         print "-----------"
681                         return "ERROR: not protected by HTTP authentication"
682
683                 if not hasattr(e, 'code') or e.code != 401:
684                         return "ERROR: failed for: %s" % str(e)
685
686                 base64data = base64.encodestring("%s:%s" % (self.username, self.password))[:-1]
687                 # NOTE: assuming basic realm authentication.
688                 authheader = "Basic %s" % base64data
689                 req.add_header("Authorization", authheader)
690
691                 try:
692                         f = urllib2.urlopen(req)
693                 except IOError, e:
694                         # failing here means the User/passwd is wrong (hopefully)
695                         raise ExceptionPassword("Incorrect username/password")
696
697                 # TODO: after verifying that the user/password is correct, we should
698                 # actually reboot the given node.
699
700                 if not dryrun:
701                         # add data to handler,
702                         # fetch url one more time on cmd.html, econtrol.html or whatever.
703                         pass
704
705                 if self.verbose: print f.read()
706
707                 self.close()
708                 return 0
709
710
711 class ePowerSwitchOld(PCUControl):
712         def run(self, node_port, dryrun):
713                 self.url = "http://%s:%d/" % (self.host,80)
714                 uri = "%s:%d" % (self.host,80)
715
716                 # create authinfo
717                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
718                 authinfo.add_password (None, uri, self.username, self.password)
719                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
720
721                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
722                 transport = urllib2.build_opener(authinfo)
723                 f = transport.open(self.url)
724                 if self.verbose: print f.read()
725
726                 if not dryrun:
727                         transport = urllib2.build_opener(authhandler)
728                         f = transport.open(self.url + "cmd.html", "P%d=r" % node_port)
729                         if self.verbose: print f.read()
730
731                 self.close()
732                 return 0
733
734 class ePowerSwitch(PCUControl):
735         def run(self, node_port, dryrun):
736                 self.url = "http://%s:%d/" % (self.host,80)
737                 uri = "%s:%d" % (self.host,80)
738
739                 # TODO: I'm still not sure what the deal is here.
740                 #               two independent calls appear to need to be made before the
741                 #               reboot will succeed.  It doesn't seem to be possible to do
742                 #               this with a single call.  I have no idea why.
743
744                 # create authinfo
745                 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
746                 authinfo.add_password (None, uri, self.username, self.password)
747                 authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
748
749                 # NOTE: it doesn't seem to matter whether this authinfo is here or not.
750                 transport = urllib2.build_opener()
751                 f = transport.open(self.url + "elogin.html", "pwd=%s" % self.password)
752                 if self.verbose: print f.read()
753
754                 if not dryrun:
755                         transport = urllib2.build_opener(authhandler)
756                         f = transport.open(self.url + "econtrol.html", "P%d=r" % node_port)
757                         if self.verbose: print f.read()
758
759                 #       data= "P%d=r" % node_port
760                 #self.open(self.host, self.username, self.password)
761                 #self.sendHTTP("elogin.html", "pwd=%s" % self.password)
762                 #self.sendHTTP("econtrol.html", data)
763                 #self.sendHTTP("cmd.html", data)
764
765                 self.close()
766                 return 0
767                 
768
769 ### rebooting european BlackBox PSE boxes
770 # Thierry Parmentelat - May 11 2005
771 # tested on 4-ports models known as PSE505-FR
772 # uses http to POST a data 'P<port>=r'
773 # relies on basic authentication within http1.0
774 # first curl-based script was
775 # curl --http1.0 --basic --user <username>:<password> --data P<port>=r \
776 #       http://<hostname>:<http_port>/cmd.html && echo OK
777
778 def bbpse_reboot (pcu_ip,username,password,port_in_pcu,http_port, dryrun):
779
780         global verbose
781
782         url = "http://%s:%d/cmd.html" % (pcu_ip,http_port)
783         data= "P%d=r" % port_in_pcu
784         if verbose:
785                 logger.debug("POSTing '%s' on %s" % (data,url))
786
787         authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm()
788         uri = "%s:%d" % (pcu_ip,http_port)
789         authinfo.add_password (None, uri, username, password)
790         authhandler = urllib2.HTTPBasicAuthHandler( authinfo )
791
792         opener = urllib2.build_opener(authhandler)
793         urllib2.install_opener(opener)
794
795         if (dryrun):
796                 return 0
797
798         try:
799                 f = urllib2.urlopen(url,data)
800
801                 r= f.read()
802                 if verbose:
803                         logger.debug(r)
804                 return 0
805
806         except urllib2.URLError,err:
807                 logger.info('Could not open http connection', err)
808                 return "bbpse error"
809
810 ### rebooting x10toggle based systems addressed by port
811 # Marc E. Fiuczynski - May 31 2005
812 # tested on 4-ports models known as PSE505-FR
813 # uses ssh and password to login to an account
814 # that will cause the system to be powercycled.
815
816 def x10toggle_reboot(ip, username, password, port, dryrun):
817         global verbose
818
819         ssh = None
820         try:
821                 ssh = pyssh.Ssh(username, ip)
822                 ssh.open()
823
824                 # Login
825                 telnet_answer(ssh, "password:", password)
826
827                 if not dryrun:
828                         # Reboot
829                         telnet_answer(ssh, "x10toggle>", "A%d" % port)
830
831                 # Close
832                 output = ssh.close()
833                 if verbose:
834                         logger.debug(output)
835                 return 0
836
837         except Exception, err:
838                 if verbose:
839                         logger.debug(err)
840                 if ssh:
841                         output = ssh.close()
842                         if verbose:
843                                 logger.debug(output)
844                 return errno.ETIMEDOUT
845
846 ### rebooting Dell systems via RAC card
847 # Marc E. Fiuczynski - June 01 2005
848 # tested with David Lowenthal's itchy/scratchy nodes at UGA
849 #
850
851 def runcmd(command, args, username, password, timeout = None):
852
853         result = [None]
854         result_ready = threading.Condition()
855
856         def set_result(x):
857
858                 result_ready.acquire()
859                 try:
860                         result[0] = x
861                 finally:
862                         result_ready.notify()
863                         result_ready.release()
864
865         def do_command(command, username, password):
866
867                 try:
868                         # Popen4 is a popen-type class that combines stdout and stderr
869                         p = popen2.Popen4(command)
870
871                         # read all output data
872                         p.tochild.write("%s\n" % username)
873                         p.tochild.write("%s\n" % password)
874                         p.tochild.close()
875                         data = p.fromchild.read()
876
877                         while True:
878                                 # might get interrupted by a signal in poll() or waitpid()
879                                 try:
880                                         retval = p.wait()
881                                         set_result((retval, data))
882                                         break
883                                 except OSError, ex:
884                                         if ex.errno == errno.EINTR:
885                                                 continue
886                                         raise ex
887                 except Exception, ex:
888                         set_result(ex)
889
890         if args:
891                 command = " ".join([command] + args)
892
893         worker = threading.Thread(target = do_command, args = (command, username, password, ))
894         worker.setDaemon(True)
895         result_ready.acquire()
896         worker.start()
897         result_ready.wait(timeout)
898         try:
899                 if result == [None]:
900                         raise Exception, "command timed-out: '%s'" % command
901         finally:
902                 result_ready.release()
903         result = result[0]
904
905         if isinstance(result, Exception):
906                 raise result
907         else:
908                 (retval, data) = result
909                 if os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == 0:
910                         return data
911                 else:
912                         out = "system command ('%s') " % command
913                         if os.WIFEXITED(retval):
914                                 out += "failed, rc = %d" % os.WEXITSTATUS(retval)
915                         else:
916                                 out += "killed by signal %d" % os.WTERMSIG(retval)
917                         if data:
918                                 out += "; output follows:\n" + data
919                         raise Exception, out
920
921 def racadm_reboot(ip, username, password, port, dryrun):
922         global verbose
923
924         try:
925                 cmd = "/usr/sbin/racadm"
926                 os.stat(cmd)
927                 if not dryrun:
928                         output = runcmd(cmd, ["-r %s -i serveraction powercycle" % ip],
929                                 username, password)
930                 else:
931                         output = runcmd(cmd, ["-r %s -i getsysinfo" % ip],
932                                 username, password)
933
934                 print "RUNCMD: %s" % output
935                 if verbose:
936                         logger.debug(output)
937                 return 0
938
939         except Exception, err:
940                 logger.debug("runcmd raised exception %s" % err)
941                 if verbose:
942                         logger.debug(err)
943                 return -1
944
945 def pcu_name(pcu):
946         if pcu['hostname'] is not None and pcu['hostname'] is not "":
947                 return pcu['hostname']
948         elif pcu['ip'] is not None and pcu['ip'] is not "":
949                 return pcu['ip']
950         else:
951                 return None
952
953 def get_pcu_values(pcu_id):
954         # TODO: obviously, this shouldn't be loaded each time...
955         import soltesz
956         fb =soltesz.dbLoad("findbadpcus")
957
958         try:
959                 values = fb['nodes']["id_%s" % pcu_id]['values']
960         except:
961                 values = None
962
963         return values
964
965 def check_open_port(values, port_list):
966         ret = False
967
968         if 'portstatus' in values:
969                 for port in port_list:
970                         if      port in values['portstatus'] and \
971                                 values['portstatus'][port] == "open":
972
973                                 ret = True
974         
975         return ret
976         
977 def reboot_policy(nodename, continue_probe, dryrun):
978         global verbose
979
980         pcu = plc.getpcu(nodename)
981         if not pcu:
982                 return False # "%s has no pcu" % nodename
983
984         values = get_pcu_values(pcu['pcu_id'])
985         if values == None:
986                 return False #"no info for pcu_id %s" % pcu['pcu_id']
987         
988         # Try the PCU first
989         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
990
991         ret = reboot_test(nodename, values, continue_probe, verbose, dryrun)
992
993         if rb_ret != 0:
994                 return False
995         else:
996                 return True
997
998 def reboot_test(nodename, values, continue_probe, verbose, dryrun):
999         rb_ret = ""
1000
1001         try:
1002                 # DataProbe iPal (many sites)
1003                 if  continue_probe and values['model'].find("Dataprobe IP-41x/IP-81x") >= 0:
1004                         ipal = IPAL(values, verbose, ['23'])
1005                         rb_ret = ipal.reboot(values[nodename], dryrun)
1006                                 
1007                 # APC Masterswitch (Berkeley)
1008                 elif continue_probe and values['model'].find("APC AP79xx/Masterswitch") >= 0:
1009
1010                         # TODO: make a more robust version of APC
1011                         if values['pcu_id'] in [1163,1055,1111,1231,1113,1127,1128,1148]:
1012                                 apc = APCEurope(values, verbose, ['22', '23'])
1013                                 rb_ret = apc.reboot(values[nodename], dryrun)
1014
1015                         elif values['pcu_id'] in [1173,1221,1220,1225]:
1016                                 apc = APCFolsom(values, verbose, ['22', '23'])
1017                                 rb_ret = apc.reboot(values[nodename], dryrun)
1018
1019                         else:
1020                                 apc = APCMaster(values, verbose, ['22', '23'])
1021                                 rb_ret = apc.reboot(values[nodename], dryrun)
1022
1023                 # BayTech DS4-RPC
1024                 elif continue_probe and values['model'].find("Baytech DS4-RPC") >= 0:
1025                         if values['pcu_id'] in [1041,1209,1025,1052,1057]:
1026                                 # These  require a 'ctrl-c' to be sent... 
1027                                 baytech = BayTechCtrlC(values, verbose, ['22', '23'])
1028                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1029
1030                         elif values['pcu_id'] in [1012]:
1031                                 # This pcu sometimes doesn't present the 'Username' prompt,
1032                                 # unless you immediately try again...
1033                                 try:
1034                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1035                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1036                                 except:
1037                                         baytech = BayTechGeorgeTown(values, verbose, ['22', '23'])
1038                                         rb_ret = baytech.reboot(values[nodename], dryrun)
1039                         else:
1040                                 baytech = BayTech(values, verbose, ['22', '23'])
1041                                 rb_ret = baytech.reboot(values[nodename], dryrun)
1042
1043                 # iLO
1044                 elif continue_probe and values['model'].find("HP iLO") >= 0:
1045                         hpilo = HPiLO(values, verbose, ['22'])
1046                         rb_ret = hpilo.reboot(0, dryrun)
1047                         if rb_ret != 0:
1048                                 hpilo = HPiLOHttps(values, verbose, ['443'])
1049                                 rb_ret = hpilo.reboot(0, dryrun)
1050
1051                 # DRAC ssh
1052                 elif continue_probe and values['model'].find("Dell RAC") >= 0:
1053                         # TODO: I don't think DRACRacAdm will throw an exception for the
1054                         # default method to catch...
1055                         try:
1056                                 drac = DRACRacAdm(values, verbose, ['443', '5869'])
1057                                 rb_ret = drac.reboot(0, dryrun)
1058                         except:
1059                                 drac = DRAC(values, verbose, ['22'])
1060                                 rb_ret = drac.reboot(0, dryrun)
1061
1062                 # BlackBox PSExxx-xx (e.g. PSE505-FR)
1063                 elif continue_probe and \
1064                         (values['model'].find("BlackBox PS5xx") >= 0 or
1065                          values['model'].find("ePowerSwitch 1/4/8x") >=0 ):
1066
1067                         # TODO: allow a different port than http 80.
1068                         if values['pcu_id'] in [1089, 1071, 1046, 1035, 1118]:
1069                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1070                         elif values['pcu_id'] in [1003]:
1071                                 eps = ePowerSwitch(values, verbose, ['80'])
1072                         else:
1073                                 eps = ePowerSwitchGood(values, verbose, ['80'])
1074
1075                         rb_ret = eps.reboot(values[nodename], dryrun)
1076
1077                 elif continue_probe:
1078                         rb_ret = "Unsupported_PCU"
1079
1080                 elif continue_probe == False:
1081                         if 'portstatus' in values:
1082                                 rb_ret = "NetDown"
1083                         else:
1084                                 rb_ret = "Not_Run"
1085                 else:
1086                         rb_ret = -1
1087
1088         except ExceptionPort, err:
1089                 rb_ret = str(err)
1090
1091         return rb_ret
1092         # ????
1093         #elif continue_probe and values['protocol'] == "racadm" and \
1094         #               values['model'] == "RAC":
1095         #       rb_ret = racadm_reboot(pcu_name(values),
1096         #                                                                 values['username'],
1097         #                                                                 values['password'],
1098         #                                                                 pcu[nodename],
1099         #                                                                 dryrun)
1100
1101 # Returns true if rebooted via PCU
1102 def reboot_old(nodename, dryrun):
1103         pcu = plc.getpcu(nodename)
1104         if not pcu:
1105                 plc.nodePOD(nodename)
1106                 return False
1107         # Try the PCU first
1108         logger.debug("Trying PCU %s %s" % (pcu['hostname'], pcu['model']))
1109
1110         # APC Masterswitch (Berkeley)
1111         if pcu['model'] == "APC Masterswitch":
1112                 err = apc_reboot(pcu['ip'], pcu['username'],pcu['password'], 
1113                                 pcu[nodename], pcu['protocol'], dryrun)
1114
1115         # DataProbe iPal (many sites)
1116         elif pcu['protocol'] == "telnet" and pcu['model'].find("IP-4") >= 0:
1117                 err = ipal_reboot(pcu['ip'],pcu['password'], pcu[nodename], dryrun)
1118
1119         # BayTech DS4-RPC
1120         elif pcu['protocol'] == "ssh" and \
1121         (pcu['model'].find("Baytech") >= 0 or pcu['model'].find("DS4") >= 0):
1122                 err = baytech_reboot(pcu['ip'], pcu['username'],pcu['password'], pcu[nodename], dryrun)
1123
1124         # BlackBox PSExxx-xx (e.g. PSE505-FR)
1125         elif pcu['protocol'] == "http" and (pcu['model'] == "bbpse"):
1126                 err = bbpse_reboot(pcu['ip'], pcu['username'], pcu['password'], pcu[nodename],80, dryrun)
1127
1128         # x10toggle
1129         elif pcu['protocol'] == "ssh" and (pcu['model'] == "x10toggle"):
1130                 err = x10toggle_reboot(pcu['ip'], pcu['username'],pcu['password'], pcu[nodename], dryrun)
1131
1132         # 
1133         elif pcu['protocol'] == "racadm" and (pcu['model'] == "RAC"):
1134                 err = racadm_reboot(pcu['ip'], pcu['username'],pcu['password'], pcu_[nodename], dryrun)
1135
1136         # Unknown or unsupported
1137         else:
1138                 err = errno.EPROTONOSUPPORT
1139                 return False
1140         return True 
1141
1142
1143 def main():
1144         logger.setLevel(logging.DEBUG)
1145         ch = logging.StreamHandler()
1146         ch.setLevel(logging.DEBUG)
1147         formatter = logging.Formatter('LOGGER - %(message)s')
1148         ch.setFormatter(formatter)
1149         logger.addHandler(ch)
1150
1151         try:
1152                 reboot("planetlab2.cs.uchicago.edu")
1153                 reboot("alice.cs.princeton.edu")
1154         except Exception, err:
1155                 print err
1156
1157 if __name__ == '__main__':
1158         import plc
1159         logger = logging.getLogger("monitor")
1160         main()