sfa/bonfire/bonfire.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 """
   4    yum -y install python-pip
   5    pip install requests
   6 """
   7 import requests
   8 import xml.etree.ElementTree as ET
   9 import subprocess
  10 import time
  11 #import ldap
  12 from ConfigParser import SafeConfigParser
  13
  14 parser = SafeConfigParser()
  15 parser.read('simple.ini')
  16
  17 """
  18     module for bonfire to connect with sfa (following the Rspec)
  19     inspired by the following documenation:
  20     https://svn.planet-lab.org/wiki/SfaDeveloperDummyTutorial#RunningSFAinDummyflavour
  21
  22     does not forget to launching this code:
  23     python /usr/lib/python2.7/site-packages/sfa/dummy/dummy_testbed_api.py
  24
  25     documentation
  26     http://wiki.bonfire-project.eu/index.php/FED4FIRE_sfa
  27
  28     1 - list resources
  29     2 - allocate
  30     3 - provisioning
  31     4 - create compute VM for fr-inria/uk-epcc
  32
  33     1) list all the resources of bonfire from sfa's point of view
  34        python -c 'import bonfire; print bonfire.bonsources()'
  35     2) allocate: create an experiment bonfire with slice information (parameters : user_name, groups, description, walltime, slice_name)
  36        python -c 'import bonfire; print bonfire.allocate("nlebreto", "nlebreto", "tdes", "125", "topdomain.dummy.nicolas")'
  37     3) provisioning: changing the status to running status for the experiment 2911
  38        python -c 'import bonfire; print bonfire.provisioning("2911","running")'
  39     4) bonfire create virtual machine with these specific features 1) fr-inria storages n°1805 network 2 ip public 2) uk-epcc storages n°1364 network 0 bonfire wan
  40        python -c 'import bonfire; print bonfire.create_vm("fr-inria", "56910", "rester", "nlebreto")'
  41        python -c 'import bonfire; print bonfire.create_vm("uk-epcc",  "56910", "rester", "nlebreto")'
  42
  43     5) retrieve the url, the name and the key for a compute N°3656 located at fr-inria
  44        python -c 'import bonfire; print bonfire.rsa_user_bonfire("fr-inria", "3656")'
  45     6) create a new user and slice for sfa wrap
  46        python -c 'import bonfire; print bonfire.new_user_slice()'
  47     7) stop virtual machine in bonfire testbed (ex n°3756 at fr-inira testbed)
  48        python -c 'import bonfire; print bonfire.stop_vm("fr-inria", "3756")'
  49     8) remove slice or key
  50        python -c 'import bonfire; print bonfire.remove_slice("topdomain.dummy.alice_slice")'
  51     9) attach slice to a user
  52        python -c 'import bonfire; print bonfire.create_slice_attach_user("topdomain.dummy.alice")'
  53     10) verify bonfire authentication
  54        python -c 'import bonfire; print bonfire.callcurl("https://api.bonfire-project.eu/")'
  55     11) bonfire ldap authentification
  56        python -c 'import bonfire; print bonfire.bonldap("nicolas.lebreton@inria.fr")'
  57 """
  58
  59 """ pseudo authentication for bonfire """
  60 def bonfire_authenticate():
  61     h = {}
  62     h["user"]      = "nlebreto"
  63     h["mail"]      = "nicolas.lebreton@inria.fr"
  64     h["user_pass"] = "GDRU_23tc$"
  65     h["location"]  = parser.get('bonfire', 'url')
  66     return h
  67
  68 """ authentification against bonfire's ldap using a virtual machine """
  69 """
  70 def bonldap(mail):
  71     ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, '/Fed4FIRE-SFA-Backend/puppet/modules/ca/ca.crt')
  72     ldap.set_option(ldap.OPT_PROTOCOL_VERSION, 3)
  73     ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW)
  74     ld = ldap.initialize('ldaps://127.0.0.1:2636')
  75     basedn = "ou=People,dc=bonfire-project,dc=eu"
  76     filter_test = "mail=" + mail
  77     filter = filter_test
  78     results = ld.search_s(basedn, ldap.SCOPE_SUBTREE, filter)
  79     if not results:
  80        print ("error 401, you need to be register to the portal f4f")
  81     return results
  82 """
  83 """ create a slice and attach a specific user to it """
  84 def create_slice_attach_user(user_slice):
  85     call = "sfa.py add -x {0}_slice -t slice -r {0}@dummy.net".format(user_slice)
  86     callcreateslice =  subprocess.Popen(call, shell=True)
  87
  88 """ remove slice or key """
  89 def remove_slice(name):
  90     cmdremove    = "sfaadmin.py reg remove {0}".format(name)
  91     removeaction = subprocess.Popen(cmdremove, shell=True)
  92
  93 """ show specific credential of a slice (see the content of a specific file  /root/.sfi/*.slice.cred) """
  94 def show_slice_credential(slice_name):
  95     path = "/root/.sfi/{0}.slice.cred".format(slice_name)
  96     tree = ET.parse(path)
  97     root = tree.getroot()
  98     hashres = {}
  99     hashres["slice_native"] = root.findall(".//signatures//{http://www.w3.org/2000/09/xmldsig#}Signature//{http://www.w3.org/2000/09/xmldsig#}KeyInfo//{http://www.w3.org/2000/09/xmldsig#}X509Data//{http://www.w3.org/2000/09/xmldsig#}X509SubjectName")[0].text
 100     hashres["X509IssuerName"] = root.findall(".//signatures//{http://www.w3.org/2000/09/xmldsig#}Signature//{http://www.w3.org/2000/09/xmldsig#}KeyInfo//{http://www.w3.org/2000/09/xmldsig#}X509Data//{http://www.w3.org/2000/09/xmldsig#}X509IssuerName")[0].text
 101     for target in root.findall('credential'):
 102         hashres["slice_user_urn"] = target.find('owner_urn').text
 103         hashres["slice_urn"] = target.find('target_urn').text
 104         hashres["serial"] = target.find('serial').text
 105     return hashres
 106
 107 """ create a bonfire experiment from a sfa point of view """
 108 def allocate(user_name, groups, description, walltime, slice_name):
 109     hashres = {}
 110     hashres = show_slice_credential(slice_name)
 111     urlexperiment = parser.get('bonfire', 'url') + "experiments"
 112     create_fed4fire_exp(user_name, groups, description, walltime, hashres["slice_urn"], hashres["slice_user_urn"], hashres["slice_native"], urlexperiment)
 113
 114 """ create a new user and slice for sfa wrap """
 115 def new_user_slice():
 116     n = rsa_user_bonfire("fr-inria", "3656")
 117     """ url = n["url"] + "." + n["name"]
 118         fix to do add -k id_rsa.pub (pb key convert) """
 119     url = "topdomain.dummy." + n["name"]
 120     txtcreateuser = "sfaadmin.py reg register -x {0} -t user -e {1}@dummy.net".format(url, n["name"])
 121     createusersfa = subprocess.Popen(txtcreateuser, shell=True)
 122     """ slicenfo = n["url"] + "." + n["name"] + "_" + n["name"] """
 123     sliceinfo = "topdomain.dummy." + n["name"] + "_slice"
 124     txtslice = "sfaadmin.py reg register -x {0} -t slice -r {1}".format(sliceinfo, url)
 125     createslice = subprocess.Popen(txtslice, shell=True)
 126
 127 """ create a experiment bonfire with the slice urn and the experiment owner """
 128 def create_fed4fire_exp(name, groups, description, walltime, slice_urn, slice_user_urn, slice_native, url_experiment_bonfire):
 129     xmldescription='<experiment xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><name>' + name +'</name><groups>' + groups + '</groups><description>' + description + '</description><walltime>' + walltime + '</walltime><status>ready</status><slice_urn>' + slice_urn + '</slice_urn><slice_usr_urn>' + slice_user_urn + '<slice_usr_urn><slice_native>' + slice_native + '</slice_native></experiment>'
 130     postexp(url_experiment_bonfire, xmldescription)
 131
 132 """ create a virtual machine with these specific features 1) fr-inria storages n°1805 network 2 ip public 2) uk-epcc storages n°1364 network 0 bonfire wan """
 133 def create_vm(testbed, nb_experiment, name_compute, groups):
 134     url = parser.get('bonfire', 'url') + 'experiments/' + num_experiment + '/computes'
 135     if testbed == "fr-inria":
 136        xmldescription='<compute xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><name>' + name_compute + '</name><groups>' + groups + '</groups><instance_type>lite</instance_type><disk><storage href="/locations/fr-inria/storages/1805"/><type>OS</type><target>hda</target></disk><nic><network href="/locations/fr-inria/networks/2"/></nic><link href="/locations/fr-inria" rel="location"/></compute>'
 137     if testbed == "uk-epcc":
 138        xmldescription='<compute xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><name>' + name_compute + '</name><groups>' + groups + '</groups><instance_type>lite</instance_type><disk><storage href="/locations/uk-epcc/storages/1364"/><type>OS</type><target>hda</target></disk><nic><network href="/locations/uk-epcc/networks/0"/></nic><link href="/locations/uk-epcc" rel="location"/></compute>'
 139     postexp(url, xmldescription)
 140
 141 """ simple post method for request """
 142 def postexp(url, xmldescription):
 143     headers = {'content-type': 'application/vnd.bonfire+xml'}
 144     h = bonfire_authenticate()
 145     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=(h["user"], h["user_pass"]))
 146
 147 """ stop a virtual machine for bonfire
 148     changing the state to stopped state """
 149 def stop_vm(testbed, num_compute):
 150     url = parser.get('bonfire', 'url') + "locations/" + testbed + "/computes/" + num_compute
 151     xmldescription = '<compute xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><state>stopped</state></compute>'
 152     headers = {'content-type': 'application/vnd.bonfire+xml'}
 153     h = bonfire_authenticate()
 154     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=(h["user"], h["user_pass"]))
 155
 156 """ provisioning : set a bonfire's experiment to running
 157     changing the status to running status """
 158 def provisioning(num_experiment, status):
 159     url = parser.get('bonfire', 'url') + "experiments/" + num_experiment
 160     xmldescription = '<experiment xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><status>' + status + '</status></experiment>'
 161     headers = {'content-type': 'application/vnd.bonfire+xml'}
 162     h = bonfire_authenticate()
 163     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=(h["user"], h["user_pass"]))
 164
 165 """ retrieving the url, the name and the keys for a specific compute """
 166 def rsa_user_bonfire(testbed, num_compute):
 167     url = parser.get('bonfire', 'url') + "locations/" + testbed + "/computes/" + num_compute
 168     pagebonfirecompute = callcurl(url)
 169     xmlreduit = ET.fromstring(pagebonfirecompute)
 170     hashres = {}
 171     hashres["url"] = url
 172     for name in xmlreduit:
 173         if name.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}groups":
 174            hashres["name"] = name.text
 175         for context in name:
 176            if context.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}authorized_keys":
 177               hashres["keys"] = context.text
 178     return hashres
 179
 180 """ do a curl request """
 181 def callcurl(url):
 182     h = bonfire_authenticate()
 183     r = requests.get(url, verify=False, auth=(h["user"], h["user_pass"]))
 184     """ with the authentication against the ldap, peuhaps these two ligns below are not necessary """
 185     if r.status_code == 401:
 186        return "error 401, you need to be register to the portal f4f"
 187     if r.status_code == 200:
 188        return r.text
 189
 190 """ create the url page """
 191 def buildpagehttp(part1, part2, locations):
 192     res = []
 193     for page in locations:
 194         res.append(part1 + page  + "/" + part2)
 195     return res
 196
 197 def boucle(itemname, xmltree, hashrspec, name):
 198     for item in xmltree.findall(itemname):
 199         hashrspec[name.text][itemname] = item.text
 200
 201 """ method to list all information from testbeds """
 202 def jfedfeat(bonfires, pageurl):
 203     pageforstatus = callcurl(pageurl)
 204     xmlreduit = ET.fromstring(pageforstatus)
 205     hashrspec = {}
 206     itemshost = ["DISK_USAGE", "MEM_USAGE", "CPU_USAGE", "MAX_DISK", "MAX_MEM",  "MAX_CPU",
 207                  "FREE_DISK",  "FREE_MEM",  "FREE_CPU", "FREE_MEM",  "FREE_CPU", "USED_DISK",
 208                  "USED_MEM",   "USED_CPU",  "RUNNING_VMS"
 209                 ]
 210     """ retrieve info for xml tree """
 211     for host in xmlreduit.findall('HOST'):
 212         for name in host.findall('NAME'):
 213             hashrspec[name.text] = {"name" : name.text}
 214             for hostshare in host.findall('HOST_SHARE'):
 215                 for itemshostname in itemshost:
 216                     boucle(itemshostname, hostshare, hashrspec, name)
 217
 218     """ jfed feature """
 219     for clef in hashrspec:
 220         bonfires.append("<node component_manager_id=\"urn:publicid:IDN+topdomain+authority+cm" +
 221                         " component_id=\"urn:publicid:IDN+topdomain:" + hashrspec[clef]["name"] +
 222                         "\" component_name=" + hashrspec[clef]["name"] + "exclusive=\"false\">" +
 223                         "  <location country=\"unknown\" longitude=\"123456\" latitude=\"654321\"/>" +
 224                         "  <interface component_id=\"urn:publicid:IDN+ple+interface+node14312:eth0\"/>" +
 225                         "  <available now=\"true\"/>" +
 226                         "  <sliver_type name=\"" + hashrspec[clef]["name"] + "\">" +
 227                         "      <bonfire:initscript name=\"" + hashrspec[clef]["name"]  + "\"/>" +
 228                         "  </sliver_type>")
 229         for infohost in itemshost:
 230             bonfires.append("  <bonfire:attribute name=\"" + infohost + "\"value=\"" + hashrspec[clef][infohost]  + "\"/>")
 231         bonfires.append("</node>")
 232
 233 """ remove the useless xml tag version """
 234 def remove_needless_txt(txt):
 235     txt=str(txt)
 236     txt=txt.replace("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n","\n")
 237     txt=txt.replace("<?xml version='1.0' encoding='UTF-8'?>\n","\n")
 238     return txt
 239
 240 """ list all bonfire resources following the sfa specification in a rspec way """
 241 def bonsources():
 242         # parameters
 243     locations = ["fr-inria", "be-ibbt", "uk-epcc"]
 244     urlnetworks = buildpagehttp(parser.get('bonfire', 'url') + "locations/", "networks", locations)
 245     urlstorages = buildpagehttp(parser.get('bonfire', 'url') + "locations/", "storages", locations)
 246     urlcomputes = buildpagehttp(parser.get('bonfire', 'url') + "locations/", "computes", locations)
 247     """ main code """
 248     bonfires = []
 249     generatedtime =  time.strftime("%FT%T%Z")
 250     sfabegin = "<RSpec type=\"SFA\" generated=" + generatedtime + "\">"
 251     bonfires.append("<?xml version=\"1.0\"?>")
 252     bonfires.append(sfabegin)
 253     bonfires.append("<managed_experiments>")
 254     manag_exp =  remove_needless_txt(callcurl(parser.get('bonfire', 'url') + "managed_experiments"))
 255     bonfires.append(manag_exp)
 256     bonfires.append("</managed_experiments><sites><machines>")
 257     jfedfeat(bonfires, "http://frontend.bonfire.grid5000.fr/one-status.xml")
 258     jfedfeat(bonfires, "http://bonfire.epcc.ed.ac.uk/one-status.xml")
 259     jfedfeat(bonfires, "http://bonfire.psnc.pl/one-status.xml")
 260     jfedfeat(bonfires, "http://nebulosus.rus.uni-stuttgart.de/one-status.xml")
 261     bonfires.append("</machines><networks>")
 262     """ adding networks information """
 263     for xmlnetworks in urlnetworks:
 264         bonfires.append(remove_needless_txt(callcurl(xmlnetworks)))
 265     bonfires.append("</networks><storages>")
 266     """ adding storages information """
 267     for xmlstorages in urlstorages:
 268         bonfires.append(remove_needless_txt(callcurl(xmlstorages)))
 269     bonfires.append("</storages><instance_types><computes>")
 270     """ adding computes information """
 271     for xmlcomputes in urlcomputes:
 272         bonfires.append(remove_needless_txt(callcurl(xmlcomputes)))
 273     bonfires.append("</computes></instance_types></sites><experiments>")
 274     exp = callcurl(parser.get('bonfire', 'url') + "experiments")
 275     rexp = remove_needless_txt(exp)
 276     bonfires.append(rexp)
 277     bonfires.append("</experiments><reservations>")
 278     """ adding reservation information """
 279     reserv = callcurl(parser.get('bonfire', 'url') + "locations/fr-inria/reservations")
 280     rreserv = remove_needless_txt(reserv)
 281     bonfires.append(rreserv)
 282     bonfires.append("</reservations>")
 283     bonfires.append("</RSpec>")
 284     bonfires = "\n".join(bonfires)
 285     bonfires = bonfires.replace("\n\n","")
 286     return bonfires