16 from openvswitch import TransformXml
17 from exceptionDefinition import FailToConnect, NodeConstraintError
18 from scp import SCPClient
21 yumOpt = "sudo -S yum -y --nogpgcheck "
23 class TransformRawXml:
24 def __init__(self, confFile = "", linkFile = "", subnet = "", prefix = "", sliceName = "", nbEnv = 1, mainKeyPriv = None, mainKeyPub = None, sliceUrl = 'http://onelab.eu', sliceDescription = 'Slice used for educationnal purpose', country = False):
25 # Definition of the api used for the actions on planetlab
26 plc_host='www.planet-lab.eu'
27 api_url='https://%s:443/PLCAPI/'%plc_host
28 self.plc_api=xmlrpclib.ServerProxy(api_url, allow_none=True)
29 # Prefix used for the slice name
32 self.sliceUrl = sliceUrl
33 self.sliceDescription = sliceDescription
35 myOpsPLC = urllib2.urlopen('http://monitor.planet-lab.org/monitor/query?hostname=on&tg_format=plain&object=nodes&nodehistory_hostname=&observed_status=on&rpmvalue=')
36 self.myOpsPLCCsv = list(csv.reader(myOpsPLC))
37 myOpsPLE = urllib2.urlopen('http://monitor.planet-lab.eu/monitor/query?hostname=on&tg_format=plain&object=nodes&nodehistory_hostname=&observed_status=on&rpmvalue=')
38 self.myOpsPLECsv = list(csv.reader(myOpsPLE))
42 self.xmlFileConf = prefix+'Conf.xml'
43 self.xmlFileLink = prefix+'Link.xml'
45 self.xmlFileConf = confFile
46 self.xmlFileLink = linkFile
47 if 'upmc_' not in sliceName:
48 self.slice_name = slice_pref+sliceName
50 self.slice_name = sliceName
52 self.nbEnv = int(nbEnv)
54 self.country = country
56 # Attribute for ssh key
57 self.mainKeyPriv = mainKeyPriv
58 self.mainKeyPub = mainKeyPub
62 # Attribute that will contain the list of environment
69 def deleteSlice(self):
70 errorCode = self.plc_api.DeleteSlice(Auth.auth, self.slice_name)
72 print "An error occured unable to delete the slice"
76 # Add 15 days to the expiration date
77 def renewSlice(self, day = 15):
79 expiration = self.plc_api.GetSlices(Auth.auth, self.slice_name, ['expires'])[0]['expires']
81 newExpiration = expiration + (3600*24*day)
82 self.plc_api.UpdateSlice(Auth.auth, self.slice_name, {'expires': newExpiration})
83 expiration = self.plc_api.GetSlices(Auth.auth, self.slice_name, ['expires'])[0]['expires']
86 # Generate a ssh key pair
87 def generateKeyPair(self,pref = "", bit = 1024):
88 key = paramiko.RSAKey.generate(bit)
89 # Save the private key file
90 keyFilePriv = pref+"id_rsa.priv"
91 key.write_private_key_file(keyFilePriv)
92 # Save the public key file
93 keyFilePub = pref+"id_rsa.pub"
94 openedFile = open(keyFilePub, 'w')
95 openedFile.write("%s %s \n" % (key.get_name(), key.get_base64()))
96 return keyFilePriv, keyFilePub
98 # press any key to continue function
99 def anykey(self, prompt="Press enter to continue...", failChars=""):
100 char = raw_input(prompt)
101 return (char not in failChars)
103 def checkSSH(self, node):
105 ssh = paramiko.SSHClient()
106 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
110 while(connected == False):
111 print "\n"+self.mainKeyPriv+" -> "+self.slice_name+"@"+node['hostname']
113 ssh.connect(node['hostname'], username = self.slice_name, key_filename = self.mainKeyPriv)
115 except paramiko.PasswordRequiredException, why:
116 print "Password required exception"
119 except paramiko.SSHException, why:
120 print "An SSH exception occured"
123 except socket.error, why:
124 print "A soccket error occured"
126 failingNode = self.plc_api.GetNodes(Auth.auth, node['id'])[0]
127 print 'FailingNode node_id -> '+str(failingNode['node_id'])
128 raise FailToConnect(failingNode['node_id'], failingNode['hostname'])
130 except Exception, why:
131 print "An error occured for host "+str(node['hostname'])+" on slice "+str(self.slice_name)
139 if i > countFailSSH and connected==False:
141 failingNode = self.plc_api.GetNodes(Auth.auth, node['id'])[0]
142 raise Exception(failingNode['hostname'])
144 if connected == False:
145 print "Now waiting 5 minutes for an update of the node"
147 except KeyboardInterrupt, why:
148 print "You interrupt the wait"
150 print "I make",i,"iteration before getting connection"
153 while(copied == False):
155 SCPClient(ssh.get_transport()).put('configService/fedora.repo')
156 SCPClient(ssh.get_transport()).put('configService/fedora-updates.repo')
157 SCPClient(ssh.get_transport()).put('configService/fedora-updates-testing.repo')
158 self.execute("sudo -S mv *.repo /etc/yum.repos.d/", ssh)
159 self.execute("sudo -S chown root:root /etc/yum.repos.d/*.repo", ssh)
161 except Exception, why:
162 print "An error occured while configuring yum"
164 if i > 4 and copied == False:
165 failingNode = self.plc_api.GetNodes(Auth.auth, node['id'])[0]
166 raise Exception(failingNode['hostname'])
174 # Install an alternate ssh server
175 def installSSH(self, userKeyFile, node):
176 ssh = self.checkSSH(node)
179 while(returnCode != 0):
181 failingNode = self.plc_api.GetNodes(Auth.auth, node['id'])[0]
182 raise Exception(failingNode['hostname'])
183 returnCode = self.execute(yumOpt+"install openssh-server", ssh, display= True)
184 print "Return Code is ->", returnCode
187 self.execute("mkdir .ssh", ssh, display= True)
188 if node['type'] == 'private':
189 SCPClient(ssh.get_transport()).put(userKeyFile, ".ssh/id_rsa.pub")
190 SCPClient(ssh.get_transport()).put("configService/sshd_config", "./")
191 elif node['type'] == 'public':
192 SCPClient(ssh.get_transport()).put(userKeyFile, ".ssh/id_rsa.pub.tmp")
193 SCPClient(ssh.get_transport()).put("configService/sshd_config", "./")
194 self.execute("cat .ssh/id_rsa.pub.tmp >> .ssh/id_rsa.pub", ssh)
195 self.execute("sudo -S mv ./sshd_config /etc/ssh/", ssh)
196 self.execute("sudo -S sh -c \"echo \\\"Port 2222\\\" >> /etc/ssh/sshd_config\"", ssh)
199 while (returnCode != 0):
201 returnCode = self.execute("sudo -S service sshd restart", ssh, display = True)
203 self.execute("sudo -S sed -i -e \"s/Port "+str(port)+"/Port "+str(port+1)+"/g\" /etc/ssh/sshd_config", ssh)
207 # Execute a command on a remote machine
208 def execute(self, command, ssh, display= False, retour= False):
210 stdin, stdout, stderr = ssh.exec_command(command)
212 while not stdout.channel.exit_status_ready():
217 splitted = err.splitlines()
218 if len(splitted) > 0:
219 print "Error in execution"
220 for line in splitted:
223 for line in stdout.read().splitlines():
227 return stdout.channel.recv_exit_status()
229 # Create the xml file corresponding to each environment
230 def createXmlFile(self, hostList, idEnv):
231 from xml.dom.minidom import parse
232 conf = parse(self.xmlFileConf)
233 link = parse(self.xmlFileLink)
234 sliceNames = link.getElementsByTagName('slice')
235 for value in sliceNames:
236 value.firstChild.nodeValue = self.slice_name
237 sliceNames = conf.getElementsByTagName('slice_name')
238 for value in sliceNames:
239 value.firstChild.nodeValue = self.slice_name
240 urlList = conf.getElementsByTagName('url')
241 if len(hostList) < len(urlList):
242 print "hostlist -> "+str(len(hostList))
243 print "urllist -> "+str(len(urlList))
244 for host in hostList:
246 for i in range(len(urlList)):
247 urlList[i].firstChild.nodeValue = hostList[i]
248 subnetIP, subnetMask = self.subnet.split('/')
249 subnetMask = int(subnetMask)
250 subnetIP = subnetIP.split('.')
251 ipList = conf.getElementsByTagName('ip')
252 for ipWithMask in ipList:
253 ip, mask = str(ipWithMask.firstChild.nodeValue).split('/')
257 if subnetMask > mask:
259 # TODO define SubnetException
261 #raise SubnetException()
263 subnetRange = subnetMask/8
264 for i in range(len(ip)):
275 ipWithMask.firstChild.nodeValue = newIP
276 confFileName = 'requ'+str(idEnv)+'Conf.xml'
277 confFile = open(confFileName, 'w')
278 conf.writexml(confFile)
280 self.envConfFile.append(confFileName)
281 linkFileName = 'requ'+str(idEnv)+'Link.xml'
282 linkFile = open(linkFileName, 'w')
283 link.writexml(linkFile)
285 self.envLinkFile.append(linkFileName)
287 def getNodeAvailable(self, site, nodeUsed = []):
288 nodes = self.plc_api.GetNodes(Auth.auth, site['node_ids'])
289 if site['peer_id'] == 1:
290 myOpsCsv = self.myOpsPLCCsv
292 myOpsCsv = self.myOpsPLECsv
295 if node['hostname'] == row[0]:
296 node['myOpsStatus'] = row[1]
300 if (node['myOpsStatus'] == 'BOOT' and node['boot_state'] == 'boot' and node['run_level'] == 'boot' and node['node_id'] not in nodeUsed):
301 nodeAvailable.append(node)
304 def getEnvironmentWithCountryConstraint(self, countryListPrivate, contryListCommon, sites, siteBanned):
305 print "Now looking for the environment using the country constraint"
308 if len(site['address_ids']) == 1:
309 sitesTmp.append(site)
311 sitesEnv = sitesCommon = []
313 # Getting the private allowed address
315 privateAddrAuth = structToArray(self.plc_api.GetAddresses(Auth.auth, {'country': countryListPrivate}, ['address_id']), 'address_id')
316 except Exception, why:
320 privateAddresses = []
323 if site['address_ids'][0] in privateAddrAuth and i < self.nbEnv:
324 privateAddresses.append(site['address_ids'][0])
326 # Getting the private Node
327 print "Getting the private node matching with the requirement"
330 if len(site['address_ids']) == 1 and site['address_ids'][0] in privateAddrAuth:
334 nodeSite = self.plc_api.GetNodes(Auth.auth, site['node_ids'], ['hostname', 'run_level', 'node_id', 'boot_state', 'hostname'])
335 for node in nodeSite:
337 if node['boot_state'] == 'boot':
339 nodeEnv.append({'type': 'private', 'id': node['node_id'], 'hostname': node['hostname']})
342 nodeStructList.append(nodeEnv)
343 sitesEnv.append(site)
344 nodeIdList = structToArray(nodeEnv, 'id')
349 print "List of site for environment :"
350 if len(nodeStructList) < self.nbEnv:
351 print "Error we are not able to find enough environment"
353 for site in sitesEnv:
356 # Getting the addresse wanted for common node
358 for country in countryListCommon:
359 address = self.plc_api.GetAddresses(Auth.auth, {'country': country}, ['address_id'])[0]['address_id']
360 if address not in privateAddresses:
361 commonAddresses.append(address)
363 # Getting the common node
366 if site['address_ids'][0] in commonAddresses:
367 nodeSite = self.plc_api.GetNodes(Auth.auth, site['node_ids'], ['hostname', 'run_level', 'node_id', 'boot_state'])
368 for node in nodeSite:
369 if node['boot_state'] == 'boot' and node['node_id'] not in nodeIdList:
370 commonNodes.append({'type': 'common','id': node['node_id'], 'hostname': node['hostname']})
371 nodeIdList.append(node['node_id'])
374 for env in nodeStructList:
375 for node in commonNodes:
379 print "Address list :"
380 #for addresse in commonAddresses:
381 # print "\t"+str(addresse)
382 print privateAddresses
383 print commonAddresses
385 return nodeStructList
387 def getEnvironmentWithDistanceConstraint(self, constraintList, sitesList, siteBanned, nodeBanned):
388 print "Now looking for the environment using the distance constraint"
390 nbPrivate = nbCommon = 0
391 nbInitPrivate = nbInitCommon = 0
392 # Counting the number of private and common node
393 # Also counting the number of node located at the same point
394 for node in constraintList:
395 if node['type'] == 'private':
397 if node['max'] == 0 and node['min'] == 0:
399 elif node['type'] == 'common' or node['type'] == 'public':
401 if node['max'] == 0 and node['min'] == 0:
404 print "Common "+str(nbCommon)+" "+str(nbInitCommon), " Private "+str(nbPrivate)+" "+str(nbInitPrivate)
405 nbReservedEnvironment = 0
409 while nbReservedEnvironment < self.nbEnv:
412 for site in sitesList:
413 if site['site_id'] not in siteUsed and site['site_id'] not in siteBanned:
415 nodeAvailable = self.getNodeAvailable(site, nodeBanned)
416 if len(nodeAvailable) > len(maxNode):
417 maxNode = nodeAvailable
419 except Exception, why:
420 print 'Couldn\'t get the node available'
424 siteUsed.append(siteInit['site_id'])
426 print "Site used -> "+str(siteUsed)
427 print siteInit['name']
428 for i in range(len(maxNode)-nbInitCommon):
430 if nbReservedEnvironment < self.nbEnv:
431 newEnv.append({'type': 'private', 'id': node['node_id'], 'hostname': node['hostname'], 'distance': 0, 'latitude': siteInit['latitude'], 'longitude': siteInit['longitude']})
432 nodePrivate.append(node['node_id'])
433 if len(newEnv) == nbInitPrivate:
434 tmpListEnv.append(newEnv)
436 nbReservedEnvironment += 1
440 for i in range(nbInitPrivate,len(constraintList)):
442 print constraintList[i]
443 if constraintList[i]['type'] in ['common', 'public']:
444 for site in sitesList:
445 if site['site_id'] not in siteBanned:
446 initPoint = geopy.Point(siteInit['latitude'], siteInit['longitude'])
447 distPoint = geopy.Point(site['latitude'], site['longitude'])
448 distance = geopy.distance.distance(initPoint, distPoint).km
449 if distance >= constraintList[i]['min'] and distance <= constraintList[i]['max']:
450 nodeAvailable = self.getNodeAvailable(site, nodePrivate+nodeBanned)
451 if len(nodeAvailable) > 0:
454 siteUsed.append(site['site_id'])
455 node = nodeAvailable[0]
456 nodeShared.append(node['node_id'])
457 commonNode.append({'type': constraintList[i]['type'], 'id': node['node_id'], 'hostname': node['hostname'], 'distance': int(distance), 'latitude': site['latitude'], 'longitude': site['longitude']})
460 if not(nodeAdded) and constraintList[i]['type'] in ['common', 'public']:
461 print 'Error no node added for constraint -> '+str(constraintList[i])
462 raise NodeConstraintError(constraintList[i], envList)
463 elif constraintList[i]['type'] == 'private':
465 for env in tmpListEnv:
466 if len(nodeToAdd) == 0:
467 for site in sitesList:
468 if site['site_id'] not in siteBanned:
469 initPoint = geopy.Point(siteInit['latitude'], siteInit['longitude'])
470 distPoint = geopy.Point(site['latitude'], site['longitude'])
471 distance = geopy.distance.distance(initPoint, distPoint).km
472 if distance >= constraintList[i]['min'] and distance <= constraintList[i]['max']:
473 nodeAvailable = self.getNodeAvailable(site, nodePrivate+nodeShared+nodeBanned)
474 if len(nodeAvailable) > 0:
475 nodeToAdd = nodeAvailable
476 node = nodeToAdd.pop(0)
477 env.append({'type': 'private', 'id': node['node_id'], 'hostname': node['hostname'], 'distance': int(distance), 'latitude': site['latitude'], 'longitude': site['longitude']})
479 for env in tmpListEnv:
480 for node in commonNode:
483 envList.extend(tmpListEnv)
487 # Reading the raw xml file to get information
488 # Return an array of array of struct containing the information concerning each environment requested
489 def readRawXml(self, siteBanned, nodeBanned):
490 print "Banned Site ->",siteBanned
491 print "Node banned ->",nodeBanned
492 from xml.dom.minidom import parse
493 conf = parse(self.xmlFileConf)
494 nbNodeByEnv = nbCommonNode = 0
497 countryListCommon = []
498 countryListPrivate = []
500 for host_i in conf.getElementsByTagName('host'):
503 hostType = host_i.getElementsByTagName('type')[0].firstChild.nodeValue
504 except Exception, why:
506 if hostType == 'private' or hostType == "":
508 elif hostType == 'common' or hostType == 'public':
511 print "Error the host type "+hostType+" doesn't exist"
515 hostCountry = host_i.getElementsByTagName('country')[0].firstChild.nodeValue
516 if hostType == 'common':
517 countryListCommon.append(hostCountry)
518 elif hostType == 'private':
519 countryListPrivate.append(hostCountry)
520 except Exception, why:
521 print "An error occured"
526 distance = host_i.getElementsByTagName('distances')[0]
527 distMin = int(distance.getElementsByTagName('min')[0].firstChild.nodeValue)
528 distMax = int(distance.getElementsByTagName('max')[0].firstChild.nodeValue)
529 constraintList.append({'type': hostType, 'min': distMin, 'max': distMax})
530 except Exception, why:
531 print "An error occured"
536 print "Getting all the site available"
537 sites = self.plc_api.GetSites(Auth.auth, ['*'])
540 nodeStructList = self.getEnvironmentWithCountryConstraint(countryListPrivate, contryListCommon, sites, siteBanned, nodeBanned)
543 nodeStructList = self.getEnvironmentWithDistanceConstraint(constraintList, sites, siteBanned, nodeBanned)
544 except NodeConstraintError as error:
545 self.nbEnv = len(error.getEnvList())
546 nodeStructList = error.getEnvList()
549 # Creating a file containing all the information about the slice created
550 origStdout = sys.stdout
551 sys.stdout = open('Test', 'w')
553 print "Slice name : "+self.slice_name
554 for env in nodeStructList:
559 for key in node.keys():
560 strNode += "\t"+str(key)+" : "+str(node[key])
563 sys.stdout = origStdout
565 return(nodeStructList)
567 def structToArray(self, requestAnswer, key):
568 #return [i[key] for i in requestAnswer]
570 for i in requestAnswer:
574 def addSliceTag(self, sliceId, tagName, tagValue):
576 returnCode = self.plc_api.AddSliceTag(Auth.auth, sliceId, tagName, tagValue)
578 print "An error Occured while adding the tag %s -> %s"% tagName, tagValue
579 except xmlrpclib.Fault as fault:
580 if fault.faultCode != 102:
581 print "An error occured"
587 sliceId = self.plc_api.GetSlices(Auth.auth, self.slice_name)[0]['slice_id']
588 except xmlrpclib.Fault, why:
589 print "An error while getting the slice already existing"
594 def createSlice(self):
596 print "Creating a new slice called : "+self.slice_name
598 sliceId = self.plc_api.AddSlice(Auth.auth, {'description': self.sliceDescription, 'name':self.slice_name, 'url': self.sliceUrl})
599 except xmlrpclib.Fault, why:
600 print "An error occured while creating the slice "+self.slice_name
602 except Exception, why:
603 print "An error occured while createing the slice "+self.slice_name
609 sliceId = self.createSlice()
612 sliceId = self.getSlice()
613 print "Adding user to the slice"
614 returnCode = self.plc_api.AddPersonToSlice(Auth.auth, Auth.auth['Username'], sliceId)
616 if type(self.nbEnv) != int:
617 print "You must enter a number"
620 print "Generating and adding the main ssh key"
622 if self.mainKeyPriv == None:
623 self.mainKeyPriv, self.mainKeyPub = self.generateKeyPair(pref = "./key/"+self.slice_name+"main", bit = 1024)
624 f = open(self.mainKeyPub)
627 keyId = self.plc_api.AddPersonKey(Auth.auth, Auth.auth['Username'], {'key_type': 'ssh', 'key': keyString})
629 print "An error occured while adding the key to the user"
631 except xmlrpclib.Fault as fault:
636 #TODO Ban failing site, waiting for fix
637 #FU Berlin || Reykjavik University || FOKUS || University of Goettingen
638 siteBanned = [7341, 7336, 443, 7059]
639 # TODO banned new lip6 node + FUNDP Site + DSCHINI
640 nodeBanned = [15953, 15954, 15955, 15956, 14810]
641 while success == False:
642 nodeList = self.readRawXml(siteBanned[:], nodeBanned)
644 print "Adding node to the slice"
652 if node['type'] == 'private' or ((node['type'] == 'common' or node['type'] == 'public') and i == 0):
653 returnCode = self.plc_api.AddSliceToNodes(Auth.auth, sliceId, [node['id']])
655 print "An error occured while adding a node to the slice"
659 except xmlrpclib.Fault as fault:
662 if j > 2 and failToAdd:
663 raise FailToConnect(node['id'], node['hostname'])
670 # Set additional ssh server using generated key file and port 2222
672 for envList in nodeList:
673 envKeyPriv, envKeyPub = self.generateKeyPair(pref = "./key/"+self.slice_name+"group"+str(i+1), bit = 1024)
674 self.envKeyPriv.append(envKeyPriv)
675 self.envKeyPub.append(envKeyPub)
677 print node['hostname']
678 if node['type'] != 'common':
679 self.installSSH(self.envKeyPub[i], node)
686 except FailToConnect as error:
687 print "Not able to get the ssh connection for node", error.getNodeUrl()
688 print 'Error node id -> '+str(error.getNodeId())
689 print 'Before adding failing node -> '+str(nodeBanned)
690 nodeBanned.append(error.getNodeId())
691 print 'After adding failing node -> '+str(nodeBanned)
692 except Exception, why:
693 print "Not able to get the ssh connection", why
697 print "Adding requested tag to the slice"
698 # Adding the tag needed :
699 # - TAG -> VALUE : DEF
702 # - vsys -> fd_tuntap : Allow to create tun/tap interface
703 # - vsys -> vroute : Allow to define new route
704 # - vsys_vnet -> SUBNET : Define the subnet allowed for tun/tap device
706 self.addSliceTag(sliceId, 'vsys', 'vif_up')
707 self.addSliceTag(sliceId, 'vsys', 'vif_down')
708 self.addSliceTag(sliceId, 'vsys', 'fd_tuntap')
709 self.addSliceTag(sliceId, 'vsys', 'vroute')
710 self.addSliceTag(sliceId, 'vsys_vnet', self.subnet)
712 # Configuring the node using the openvswitch.py script
713 # Command used to execute the script
718 hostnameList = self.structToArray(env, 'hostname')
719 self.createXmlFile(hostnameList, i)
722 x = TransformXml(prefix = 'requ'+str(i), keyFile = self.mainKeyPriv)
727 self.envList.append(x.getSliceList())
728 except Exception, why:
729 print "An error occured while configuring the environment number "+str(i-1)
736 if __name__ == "__main__":
737 parser = argparse.ArgumentParser()
738 parser.add_argument("-s", "--subnet", help="The subnet reserved for the slice, it must include at least all the ip address used by the tp. The syntax is SUBNET/PREFIX (e.g 10.1.0.0/16 which is the default value)", default='10.1.0.0/16')
739 parser.add_argument("-n", "--name", help="Name of the slice", default='tp')
740 parser.add_argument("-nb", "--number", help="Number of the environment you want", type=int, default=1)
741 groupInput = parser.add_mutually_exclusive_group()
742 groupInput.add_argument('-f', '--file', help="The xml file that will be used, Link xml file then Configuration xml file", type=str, default='', nargs=2)
743 groupInput.add_argument("-p", "--prefix", help="A prefix used to find the xml file, the script will look at prefix+Conf.xml and prefix+Link.xml", type=str, dest='prefix')
746 args = parser.parse_args()
747 except Exception, why:
748 print "An error occured while parsing the argument"
754 x = TransformRawXml(confFile = args.file[0], linkFile = args.file[1], subnet = args.subnet, sliceName = args.name, nbEnv = args.number)
757 x = TransformRawXml(prefix = args.prefix, subnet = args.subnet, sliceName = args.name, nbEnv = args.number)