node_ssh/nst.py

   1 #!/usr/bin/python
   2
   3 import time, sys, urllib, os, tempfile, random
   4 import xmlrpclib
   5 from optparse import OptionParser
   6 from getpass import getpass
   7 from time import sleep
   8
   9 parser = OptionParser()
  10 parser.add_option("-c", "--config", action="store", dest="config",  help="Path to alternate config file")
  11 parser.add_option("-x", "--url", action="store", dest="url", help = "API URL")
  12 parser.add_option("-s", "--slice", action="store", dest="slice", help = "Name of slice to use")
  13 parser.add_option("-n", "--nodes", action="store", dest="nodes", help = "File that contains a list of nodes to try to access")
  14 parser.add_option("-k", "--key", action="store", dest="key", help = "Path to alternate public key")
  15 parser.add_option("-u", "--user", action="store", dest="user", help = "API user name")
  16 parser.add_option("-p", "--password", action="store", dest="password", help = "API password")
  17 parser.add_option("-g", "--graph-only", action="store_true", dest="graph_only", help = "Only plot the current data, then exit")
  18 parser.add_option("-l", "--plot-length", action="store", dest="plot_length", help = "Plot x-axis (time) length in seconds")
  19 parser.add_option("-v", "--verbose", action="store_true",  dest="verbose", help="Be verbose (default: %default)")
  20 (options, args) = parser.parse_args()
  21
  22 # If user is specified but password is not
  23 if options.user is not None and options.password is None:
  24     try:
  25         options.password = getpass()
  26     except (EOFError, KeyboardInterrupt):
  27         print
  28         sys.exit(0)
  29
  30 class Config:
  31
  32     def __init__(self, options):
  33
  34         # if options are specified use them
  35         # otherwise use options from config file
  36         if options.config: config_file = options.config
  37         else: config_file = '/usr/share/planetlab/tests/node-ssh/nst_config'
  38
  39         try:
  40             execfile(config_file, self.__dict__)
  41         except:
  42             raise "Could not find nst config in " + config_file
  43
  44         if options.url: self.url = self.NST_API_SERVER = options.url
  45         if options.slice: self.NST_SLICE = options.slice
  46         if options.key: self.NST_KEY_PATH = options.key
  47         if options.user: self.NST_USER = options.user
  48         if options.password: self.NST_PASSWORD = options.password
  49         if options.nodes: self.NST_NODES = options.nodes
  50         else: self.NST_NODES = None
  51         if options.plot_length: self.NST_PLOT_LENGTH = options.plot_length
  52
  53         self.api = xmlrpclib.Server(self.NST_API_SERVER)
  54         self.auth = {}
  55         self.auth['Username'] = self.NST_USER
  56         self.auth['AuthString'] = self.NST_PASSWORD
  57         self.auth['AuthMethod'] = 'password'
  58         self.key = self.NST_KEY_PATH
  59         self.slice = self.NST_SLICE
  60         self.nodes = self.NST_NODES
  61         self.plot_length = self.NST_PLOT_LENGTH
  62         self.sleep_time = 900
  63         self.verbose = options.verbose
  64
  65         # set up directories
  66         self.data_path = '/var/lib/planetlab/tests/node-ssh/data/'
  67         self.plots_path = '/var/lib/planetlab/tests/node-ssh/plots/'
  68
  69         # set up files
  70         self.all_nodes_filename = self.data_path + os.sep + "nodes"
  71         self.nodes_in_slice_filename = self.data_path + os.sep + "nodes_in_slice"
  72         self.nodes_can_ssh_filename = self.data_path + os.sep + "nodes_can_ssh"
  73         self.nodes_good_comon_filename = self.data_path + os.sep + "nodes_good"
  74
  75
  76 # get formatted tic string for gnuplot
  77 def getTimeTicString(t1, t2, step):
  78         first_hour = list(time.localtime(t1))
  79         if not first_hour[4] == first_hour[5] == 0:
  80                 first_hour[4] = 0
  81                 first_hour[5] = 0
  82
  83         first_hour_time = int(time.mktime(first_hour))
  84         first_hour_time += 3600
  85
  86         backsteps = (first_hour_time - t1)
  87         backsteps /= step
  88         start = first_hour_time - backsteps * step
  89
  90         tics = []
  91         thistime = start
  92         while thistime < t2:
  93                 tics.append("\"%s\" %d" % \
  94                         (time.strftime("%H:%M", time.localtime(thistime)), thistime))
  95                 thistime += step
  96
  97         ticstr = ", ".join(tics)
  98         return ticstr
  99
 100
 101 # count total number of nodes in PlanetLab, according to the api
 102 # count total number  of nodes in slice, according to the api
 103 def count_nodes_by_api(config, current_time, all_nodes):
 104
 105         # count all nodes
 106         all_nodes_output = "%d\t%d" % (current_time, len(all_nodes))
 107
 108         # count all nodes in slice
 109         if config.slice == 'root':
 110             nodes_in_slice = all_nodes
 111             nodes_in_slice_output = all_nodes_output
 112         else:
 113             slice_id =config.api.GetSlices(config.auth, {'name': config.slice}, ['slice_id'])[0]['slice_id']
 114             nodes_in_slice = [row['node_id'] for row in \
 115                               all_nodes if slice_id in row['slice_ids']]
 116             nodes_in_slice_output =  "%d\t%d" % (current_time, len(nodes_in_slice))
 117
 118         # write result to datafiles
 119         all_nodes_file = open(config.all_nodes_filename, 'a')
 120         all_nodes_file.write(all_nodes_output + "\n")
 121         all_nodes_file.close()
 122
 123         nodes_in_slice_file = open(config.nodes_in_slice_filename, 'a')
 124         nodes_in_slice_file.write(nodes_in_slice_output + "\n")
 125         nodes_in_slice_file.close()
 126
 127         if config.verbose:
 128             print "all node: " + all_nodes_output
 129             print "nodes in slice: " + nodes_in_slice_output
 130
 131
 132 # count total number of "good" nodes, according to CoMon
 133 def count_nodes_good_by_comon(config, current_time):
 134
 135
 136         comon = urllib.urlopen("http://summer.cs.princeton.edu/status/tabulator.cgi?table=table_nodeviewshort&format=nameonly&select='resptime%20%3E%200%20&&%20((drift%20%3E%201m%20||%20(dns1udp%20%3E%2080%20&&%20dns2udp%20%3E%2080)%20||%20gbfree%20%3C%205%20||%20sshstatus%20%3E%202h)%20==%200)'")
 137         good_nodes = comon.readlines()
 138
 139         comon_output =  "%d\t%d" % (current_time, len(good_nodes))
 140         nodes_good_comon_file = open(config.nodes_good_comon_filename, 'a')
 141         nodes_good_comon_file.write(comon_output + "\n")
 142         nodes_good_comon_file.close()
 143
 144         if config.verbose:
 145             print "comon: " + comon_output
 146
 147 # count total number of nodes reachable by ssh
 148 def count_nodes_can_ssh(config, current_time, all_nodes):
 149
 150         api = config.api
 151         slice = config.slice
 152         key = config.key
 153         verbose = config.verbose
 154         auth = config.auth
 155         nodes = config.nodes
 156
 157         if verbose:
 158             verbose_text = ""
 159             print "Creating list of nodes to ssh to"
 160         else:
 161             verbose_text = ">/dev/null 2>&1"
 162
 163         # creaet node dict
 164         node_dict = {}
 165         for node in all_nodes:
 166             node_dict[node['hostname']] = node
 167
 168         # create node list
 169         if nodes:
 170             nodes_file = open(nodes, 'r')
 171             nodes_filename = nodes_file.name
 172             lines = nodes_file.readlines()
 173             node_list = [node.replace('\n', '') for node in lines]
 174             nodes_file.close()
 175
 176         else:
 177             node_list = node_dict.keys()
 178             nodes_filename = tempfile.mktemp()
 179             nodes_file = open(nodes_filename, 'w')
 180             for node in node_list:
 181                 nodes_file.write("%(node)s\n" % locals())
 182             nodes_file.close()
 183
 184         # creaet node dict
 185         node_dict = {}
 186         for node in all_nodes:
 187             node_dict[node['hostname']] = node
 188
 189         private_key = key.split(".pub")[0]
 190
 191         # create ssh command
 192         if verbose:
 193             print "Attemptng to ssh to nodes in " + nodes_filename
 194
 195         ssh_filename = tempfile.mktemp()
 196         ssh_file = open(ssh_filename, 'w')
 197         ssh_file.write("""
 198         export MQ_SLICE="%(slice)s"
 199         export MQ_NODES="%(nodes_filename)s"
 200
 201         eval `ssh-agent` >/dev/null 2>&1
 202         trap "kill $SSH_AGENT_PID" 0
 203         ssh-add %(private_key)s >/dev/null 2>&1
 204
 205         multiquery 'hostname' 2>/dev/null |
 206         grep "bytes" |
 207         grep -v ": 0 bytes"
 208         """ % locals())
 209         ssh_file.close()
 210         ssh_results = os.popen("bash %(ssh_filename)s" % locals()).readlines()
 211         good_nodes= [result.split(':')[0] for result in ssh_results]
 212
 213         # remove temp files
 214         if os.path.exists(nodes_filename): os.unlink(nodes_filename)
 215         if os.path.exists(ssh_filename): os.unlink(ssh_filename)
 216
 217         # count number of node we can ssh into
 218         ssh_count = len(good_nodes)
 219
 220         # determine whince nodes are dead:
 221         dead_nodes = set(node_list).difference(good_nodes)
 222
 223         # write dead nodes to file
 224         dead_node_count_output = "%d\t%d" % (current_time, len(dead_nodes))
 225         dead_nodes_file_name = config.data_path + os.sep + "dead_nodes"
 226         dead_nodes_file = open(dead_nodes_file_name, 'w')
 227
 228         for hostname in dead_nodes:
 229             boot_state = node_dict[hostname]['boot_state']
 230             last_contact = 0
 231             if node_dict[hostname]['last_contact']:
 232                 last_contact = node_dict[hostname]['last_contact']
 233             dead_nodes_file.write("%(current_time)d\t%(hostname)s\t%(boot_state)s\t%(last_contact)d\n" % \
 234                                   locals())
 235         dead_nodes_file.close()
 236
 237         # write good node count
 238         ssh_result_output =  "%d\t%d" % (current_time, ssh_count)
 239         nodes_can_ssh_file = open(config.nodes_can_ssh_filename, 'a')
 240         nodes_can_ssh_file.write(ssh_result_output + "\n")
 241         nodes_can_ssh_file.close()
 242
 243         if verbose:
 244             print "nodes that can ssh: " + ssh_result_output
 245             print "dead nodes: " + dead_node_count_output
 246
 247
 248 # remove all nodes from a slice
 249 def empty_slice(config, all_nodes):
 250
 251         if config.verbose:
 252             print "Removing %s from all nodes" % config.slice
 253
 254         all_node_ids = [row['node_id'] for row in all_nodes]
 255         config.api.DeleteSliceFromNodes(config.auth, config.slice, all_node_ids)
 256
 257
 258 # add slice to all nodes.
 259 # make sure users key is up to date
 260 def init_slice(config, all_nodes):
 261
 262     api  = config.api
 263     auth = config.auth
 264     slice = config.slice
 265     key_path = config.key
 266     verbose = config.verbose
 267     slices = api.GetSlices(auth, [slice], \
 268                                   ['slice_id', 'name', 'person_ids', 'node_ids'])
 269     if not slices:
 270         raise "No such slice %s" % slice
 271     slice = slices[0]
 272
 273     # make sure user is in slice
 274     person = api.GetPersons(auth, auth['Username'], \
 275                                    ['person_id', 'email', 'slice_ids', 'key_ids'])[0]
 276     if slice['slice_id'] not in person['slice_ids']:
 277         raise "%s not in %s slice. Must be added first" % \
 278               (person['email'], slice['name'])
 279
 280     # make sure user key is up to date
 281     current_key = open(key_path, 'r').readline().strip()
 282     if len(current_key) == 0:
 283         raise "Key cannot be empty"
 284
 285     keys = api.GetKeys(auth, person['key_ids'])
 286     if not keys:
 287         if verbose:
 288             print "Adding new key " + key_path
 289         api.AddPersonKey(auth, person['person_id'], \
 290                                 {'key_type': 'ssh', 'key': current_key})
 291
 292     elif not filter(lambda k: k['key'] == current_key, keys):
 293         if verbose:
 294             print "%s was modified or is new. Updating PLC"
 295         old_key = keys[0]
 296         api.UpdateKey(auth, old_key['key_id'], \
 297                              {'key': current_key})
 298
 299     # add slice to all nodes
 300     if verbose:
 301         print "Generating list of all nodes not in slice"
 302     all_node_ids = [row['node_id'] for row in all_nodes]
 303
 304     new_nodes = set(all_node_ids).difference(slice['node_ids'])
 305     if verbose:
 306         print "Adding %s to nodes: %r " % (slice['name'], new_nodes)
 307
 308     api.AddSliceToNodes(auth, slice['slice_id'], list(new_nodes))
 309
 310
 311 # create the fill/empty plot
 312 def plot_fill_empty(config):
 313         #ticstep = 3600 # 1 hour
 314         #plotlength = 36000 # 10 hours
 315         ticstep = 3600
 316         plotlength = int(config.plot_length)
 317         plots_path = config.plots_path
 318
 319         all_nodes_filename = config.all_nodes_filename
 320         nodes_in_slice_filename = config.nodes_in_slice_filename
 321         nodes_can_ssh_filename = config.nodes_can_ssh_filename
 322         nodes_good_comon_filename = config.nodes_good_comon_filename
 323
 324         tmpfilename = tempfile.mktemp()
 325         tmpfile = open(tmpfilename, 'w')
 326
 327         starttime = -1
 328         stoptime = -1
 329         for datafilename in [all_nodes_filename,
 330                              nodes_in_slice_filename, \
 331                              nodes_can_ssh_filename, \
 332                              nodes_good_comon_filename]:
 333                 datafile = open(datafilename, 'r')
 334                 lines = datafile.readlines()
 335                 if len(lines) > 0:
 336                     line_start = lines[0]
 337                     line_end = lines[len(lines) -1]
 338                 else:
 339                     continue
 340
 341                 thisstarttime = int(line_start.split("\t")[0])
 342                 if starttime == -1 or thisstarttime < starttime:
 343                         starttime = thisstarttime
 344                 thisstoptime = int(line_end.split("\t")[0])
 345                 if stoptime == -1 or thisstoptime > stoptime:
 346                         stoptime = thisstoptime
 347
 348         stopx = stoptime
 349         startx = max(starttime, stopx - plotlength)
 350         starttime = startx
 351
 352         tics = getTimeTicString(starttime, stoptime, ticstep)
 353
 354         startdate = time.strftime("%b %m, %Y - %H:%M", time.localtime(startx))
 355         stopdate = time.strftime("%H:%M", time.localtime(stopx))
 356
 357         if config.verbose:
 358             print "plotting data with start date: %(startdate)s and stop date: %(stopdate)s" % locals()
 359
 360         plot_output ="""
 361         set term png
 362         set output "%(plots_path)s/fill_empty.png"
 363
 364         set title "Number of Nodes / Time - %(startdate)s to %(stopdate)s"
 365         set xlabel "Time"
 366         set ylabel "Number of Nodes"
 367
 368         set xtics (%(tics)s)
 369         set xrange[%(startx)d:%(stopx)d]
 370         set yrange[0:950]
 371
 372         plot "%(all_nodes_filename)s" u 1:2 w lines title "Total Nodes", \
 373              "%(nodes_in_slice_filename)s" u 1:2 w lines title "Nodes in Slice", \
 374              "%(nodes_good_comon_filename)s" u 1:2 w lines title \
 375                         "Healthy Nodes (according to CoMon)", \
 376              "%(nodes_can_ssh_filename)s" u 1:2 w lines title "Nodes Reachable by SSH"
 377
 378         """
 379         tmpfile.write(plot_output % locals())
 380         tmpfile.close()
 381
 382         if config.verbose:
 383             print plot_output % locals()
 384
 385         os.system("gnuplot %s" %  tmpfilename)
 386
 387         if os.path.exists(tmpfilename): os.unlink(tmpfilename)
 388
 389
 390
 391 # load configuration
 392 config = Config(options)
 393
 394 if options.graph_only:
 395     plot_fill_empty(config)
 396     sys.exit(0)
 397
 398 current_time = round(time.time())
 399 all_nodes = config.api.GetNodes(config.auth, {}, \
 400                                 ['node_id', 'boot_state', 'hostname', 'last_contact', 'slice_ids'])
 401
 402
 403 # if root is specified we will ssh into root context, not a slice
 404 # so no need to add a slice to all nodes
 405 if config.slice == 'root':
 406
 407     if config.verbose:
 408         print "Logging in as root"
 409 else:
 410     # set up slice and add it to nodes
 411     init_slice(config, all_nodes)
 412
 413     if config.verbose:
 414         print "Waiting %d seconds for nodes to update" % config.sleep_time
 415
 416     # wait for nodes to get the data
 417     sleep(config.sleep_time)
 418
 419
 420 # gather data
 421 count_nodes_can_ssh(config, current_time, all_nodes)
 422 count_nodes_by_api(config, current_time, all_nodes)
 423 count_nodes_good_by_comon(config, current_time)
 424
 425 # update plots
 426 plot_fill_empty(config)
 427 #os.system("cp plots/*.png ~/public_html/planetlab/tests")
 428
 429 # clean up
 430 empty_slice(config, all_nodes)
 431