sfa/bonfire/bonfire.py

   1 #!/usr/bin/python
   2 # -*- coding:utf-8 -*-
   3 #yum -y install python-pip
   4 #pip install requests
   5 import requests
   6 import xml.etree.ElementTree as ET
   7 # parsing a xml content
   8 import subprocess
   9 # using for calling sfa code
  10
  11 import time
  12 #import ldap
  13 # will be use to authentice against bonfire's ldap
  14
  15 # module for bonfire to connect with sfa (following the Rspec)
  16 # inspired by the following documenation:
  17 # https://svn.planet-lab.org/wiki/SfaDeveloperDummyTutorial#RunningSFAinDummyflavour
  18
  19 # does not forget to launching this code:
  20 #python /usr/lib/python2.7/site-packages/sfa/dummy/dummy_testbed_api.py
  21
  22 # documentation
  23 # http://wiki.bonfire-project.eu/index.php/FED4FIRE_sfa
  24
  25 #1 - list resources
  26 #2 - allocate
  27 #3 - provisioning
  28 #4 - create compute VM for fr-inria/uk-epcc
  29
  30 # 1) list all the resources of bonfire from sfa's point of view
  31 # python -c 'import bonfire; print bonfire.bonsources()'
  32
  33 # 2) allocate: create an experiment bonfire with slice information (parameters : user_name, groups, description, walltime, slice_name)
  34 # python -c 'import bonfire; print bonfire.allocate("nlebreto", "nlebreto", "tdes", "125", "topdomain.dummy.nicolas")'
  35
  36 # 3) provisioning: changing the status to running status for the experiment 2911
  37 # python -c 'import bonfire; print bonfire.provisioning("2911","running")'
  38
  39 # 4) bonfire create virtual machine with these specific features 1) fr-inria storages n°1805 network 2 ip public 2) uk-epcc storages n°1364 network 0 bonfire wan
  40 # python -c 'import bonfire; print bonfire.create_vm("fr-inria", "56910", "rester", "nlebreto")'
  41 # python -c 'import bonfire; print bonfire.create_vm("uk-epcc",  "56910", "rester", "nlebreto")'
  42
  43 # 5) retrieve the url, the name and the key for a compute N°3656 located at fr-inria
  44 # python -c 'import bonfire; print bonfire.rsa_user_bonfire("fr-inria", "3656")'
  45
  46 # 6) create a new user and slice for sfa wrap
  47 # python -c 'import bonfire; print bonfire.new_user_slice()'
  48
  49 # 7) stop virtual machine in bonfire testbed (ex n°3756 at fr-inira testbed)
  50 # python -c 'import bonfire; print bonfire.stop_vm("fr-inria", "3756")'
  51
  52 # 8) remove slice or key
  53 # python -c 'import bonfire; print bonfire.remove_slice("topdomain.dummy.alice_slice")'
  54
  55 # 9) attach slice to a user
  56 # python -c 'import bonfire; print bonfire.create_slice_attach_user("topdomain.dummy.alice")'
  57
  58 # 10) verify bonfire authentication
  59 # python -c 'import bonfire; print bonfire.callcurl("https://api.bonfire-project.eu/")'
  60
  61 # 11) bonfire ldap authentification
  62 # python -c 'import bonfire; print bonfire.bonldap("nicolas.lebreton@inria.fr")'
  63
  64 # ########################################################## #
  65 # ########################################################## #
  66
  67 # pseudo authentication for bonfire
  68 def bonfire_authenticate():
  69     h = {}
  70     h["user"]      = "nlebreto"
  71     h["mail"]      = "nicolas.lebreton@inria.fr"
  72     h["user_pass"] = "GDRU_23tc$"
  73     h["location"]  = "https://api.integration.bonfire.grid5000.fr"
  74     return h
  75
  76 # authentification against bonfire's ldap using a virtual machine
  77 #def bonldap(mail):
  78 #    ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, '/Fed4FIRE-SFA-Backend/puppet/modules/ca/ca.crt')
  79     # using a certificate (client)
  80 #    ldap.set_option(ldap.OPT_PROTOCOL_VERSION, 3)
  81 #    ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW)
  82 #    ld = ldap.initialize('ldaps://127.0.0.1:2636')
  83     # connection with ldaps
  84 #    basedn = "ou=People,dc=bonfire-project,dc=eu"
  85 #    filter_test = "mail=" + mail
  86 #    filter = filter_test
  87     # search email in the ldap
  88 #    results = ld.search_s(basedn, ldap.SCOPE_SUBTREE, filter)
  89 #    if not results:
  90 #       print ("error 401, you need to be register to the portal f4f")
  91 #    return results
  92
  93 # create a slice and attach a specific user to it
  94 def create_slice_attach_user(user_slice):
  95     call = "sfa.py add -x {0}_slice -t slice -r {0}@dummy.net".format(user_slice)
  96     callcreateslice =  subprocess.Popen(call, shell=True)
  97
  98 # remove slice or key
  99 def remove_slice(name):
 100     cmdremove    = "sfaadmin.py reg remove {0}".format(name)
 101     removeaction = subprocess.Popen(cmdremove, shell=True)
 102
 103 # show specific credential of a slice (see the content of a specific file  /root/.sfi/*.slice.cred)
 104 def show_slice_credential(slice_name):
 105     path = "/root/.sfi/{0}.slice.cred".format(slice_name)
 106     tree = ET.parse(path)
 107     root = tree.getroot()
 108     hash = {}
 109     hash["slice_native"] = root.findall(".//signatures//{http://www.w3.org/2000/09/xmldsig#}Signature//{http://www.w3.org/2000/09/xmldsig#}KeyInfo//{http://www.w3.org/2000/09/xmldsig#}X509Data//{http://www.w3.org/2000/09/xmldsig#}X509SubjectName")[0].text
 110     hash["X509IssuerName"] = root.findall(".//signatures//{http://www.w3.org/2000/09/xmldsig#}Signature//{http://www.w3.org/2000/09/xmldsig#}KeyInfo//{http://www.w3.org/2000/09/xmldsig#}X509Data//{http://www.w3.org/2000/09/xmldsig#}X509IssuerName")[0].text
 111     for target in root.findall('credential'):
 112         hash["slice_user_urn"] = target.find('owner_urn').text
 113         hash["slice_urn"] = target.find('target_urn').text
 114         hash["serial"] = target.find('serial').text
 115     return hash
 116
 117 # create a bonfire experiment from a sfa point of view
 118 def allocate(user_name, groups, description, walltime, slice_name):
 119     hash ={}
 120     hash = show_slice_credential(slice_name)
 121     create_fed4fire_exp(user_name, groups, description, walltime, hash["slice_urn"], hash["slice_user_urn"], hash["slice_native"], "https://api.integration.bonfire.grid5000.fr/experiments")
 122
 123 # create a new user and slice for sfa wrap
 124 def new_user_slice():
 125     n = rsa_user_bonfire("fr-inria", "3656")
 126     #url = n["url"] + "." + n["name"]
 127     # fix to do add -k id_rsa.pub (pb key convert)
 128     url = "topdomain.dummy." + n["name"]
 129     txtcreateuser = "sfaadmin.py reg register -x {0} -t user -e {1}@dummy.net".format(url, n["name"])
 130     createusersfa = subprocess.Popen(txtcreateuser, shell=True)
 131     #slice = n["url"] + "." + n["name"] + "_" + n["name"]
 132     slice = "topdomain.dummy." + n["name"] + "_slice"
 133     txtslice = "sfaadmin.py reg register -x {0} -t slice -r {1}".format(slice, url)
 134     createslice = subprocess.Popen(txtslice, shell=True)
 135
 136 # create a experiment bonfire with the slice urn and the experiment owner
 137 def create_fed4fire_exp(name, groups, description, walltime, slice_urn, slice_user_urn, slice_native, url_experiment_bonfire):
 138     xmldescription='<experiment xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><name>' + name +'</name><groups>' + groups + '</groups><description>' + description + '</description><walltime>' + walltime + '</walltime><status>ready</status><slice_urn>' + slice_urn + '</slice_urn><slice_usr_urn>' + slice_user_urn + '<slice_usr_urn><slice_native>' + slice_native + '</slice_native></experiment>'
 139     postexp(url_experiment_bonfire, xmldescription)
 140
 141 # create a virtual machine with these specific features 1) fr-inria storages n°1805 network 2 ip public 2) uk-epcc storages n°1364 network 0 bonfire wan
 142 def create_vm(testbed, nb_experiment, name_compute, groups):
 143     url = 'https://api.integration.bonfire.grid5000.fr/experiments/' + num_experiment + '/computes'
 144     if testbed == "fr-inria":
 145        xmldescription='<compute xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><name>' + name_compute + '</name><groups>' + groups + '</groups><instance_type>lite</instance_type><disk><storage href="/locations/fr-inria/storages/1805"/><type>OS</type><target>hda</target></disk><nic><network href="/locations/fr-inria/networks/2"/></nic><link href="/locations/fr-inria" rel="location"/></compute>'
 146     if testbed == "uk-epcc":
 147        xmldescription='<compute xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><name>' + name_compute + '</name><groups>' + groups + '</groups><instance_type>lite</instance_type><disk><storage href="/locations/uk-epcc/storages/1364"/><type>OS</type><target>hda</target></disk><nic><network href="/locations/uk-epcc/networks/0"/></nic><link href="/locations/uk-epcc" rel="location"/></compute>'
 148     postexp(url, xmldescription)
 149
 150 # simple post method for request
 151 def postexp(url, xmldescription):
 152     headers = {'content-type': 'application/vnd.bonfire+xml'}
 153     h = bonfire_authenticate()
 154     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=(h["user"], h["user_pass"]))
 155
 156 # stop a virtual machine for bonfire
 157 # changing the state to stopped state
 158 def stop_vm(testbed, num_compute):
 159     url = "https://api.integration.bonfire.grid5000.fr/" + "locations/" + testbed + "/computes/" + num_compute
 160     xmldescription = '<compute xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><state>stopped</state></compute>'
 161     headers = {'content-type': 'application/vnd.bonfire+xml'}
 162     h = bonfire_authenticate()
 163     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=(h["user"], h["user_pass"]))
 164
 165 # provisioning : set a bonfire's experiment to running
 166 # changing the status to running status
 167 def provisioning(num_experiment, status):
 168     url = "https://api.integration.bonfire.grid5000.fr/experiments/" + num_experiment
 169     xmldescription = '<experiment xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><status>'+ status + '</status></experiment>'
 170     headers = {'content-type': 'application/vnd.bonfire+xml'}
 171     h = bonfire_authenticate()
 172     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=(h["user"], h["user_pass"]))
 173
 174 # retrieving the url, the name and the keys for a specific compute
 175 def rsa_user_bonfire(testbed, num_compute):
 176     url = "https://api.integration.bonfire.grid5000.fr/" + "locations/" + testbed + "/computes/" + num_compute
 177     pagebonfirecompute = callcurl(url)
 178     xmlreduit = ET.fromstring(pagebonfirecompute)
 179     hash = {}
 180     hash["url"] = url
 181     for name in xmlreduit:
 182         if name.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}groups":
 183            hash["name"] = name.text
 184         for context in name:
 185            if context.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}authorized_keys":
 186               hash["keys"] = context.text
 187     return hash
 188
 189 # do a curl request
 190 def callcurl(url):
 191     h = bonfire_authenticate()
 192     r = requests.get(url, verify=False, auth=(h["user"], h["user_pass"]))
 193     # with the authentication against the ldap, peuhaps these two ligns below is not necessary
 194     if r.status_code == 401:
 195        return "error 401, you need to be register to the portal f4f"
 196     if r.status_code == 200:
 197        return r.text
 198
 199 # create the url page
 200 def buildpagehttp(part1, part2, locations):
 201     res = []
 202     for page in locations:
 203         res.append(part1 + page  + "/" + part2)
 204     return res
 205
 206 def boucle(itemname, xmltree, hashrspec, name):
 207     for item in xmltree.findall(itemname):
 208         hashrspec[name.text][itemname] = item.text
 209
 210 # method to list all information from testbeds
 211 def jfedfeat(bonfires, pageurl):
 212     pageforstatus = callcurl(pageurl)
 213     xmlreduit = ET.fromstring(pageforstatus)
 214     hashrspec = {}
 215     itemshost = ["DISK_USAGE", "MEM_USAGE", "CPU_USAGE", "MAX_DISK", "MAX_MEM",  "MAX_CPU",
 216                  "FREE_DISK",  "FREE_MEM",  "FREE_CPU", "FREE_MEM",  "FREE_CPU", "USED_DISK",
 217                  "USED_MEM",   "USED_CPU",  "RUNNING_VMS"
 218                 ]
 219     # retrieve info for xml tree
 220     for host in xmlreduit.findall('HOST'):
 221         for name in host.findall('NAME'):
 222             hashrspec[name.text] = {"name" : name.text}
 223             for hostshare in host.findall('HOST_SHARE'):
 224                 for itemshostname in itemshost:
 225                     boucle(itemshostname, hostshare, hashrspec, name)
 226
 227  # jfed feature
 228     for clef in hashrspec:
 229         bonfires.append("<node component_manager_id=\"urn:publicid:IDN+topdomain+authority+cm" +
 230                         " component_id=\"urn:publicid:IDN+topdomain:" + hashrspec[clef]["name"] +
 231                         "\" component_name=" + hashrspec[clef]["name"] + "exclusive=\"false\">" +
 232                         "  <location country=\"unknown\" longitude=\"123456\" latitude=\"654321\"/>" +
 233                         "  <interface component_id=\"urn:publicid:IDN+ple+interface+node14312:eth0\"/>" +
 234                         "  <available now=\"true\"/>" +
 235                         "  <sliver_type name=\"" + hashrspec[clef]["name"] + "\">" +
 236                         "      <bonfire:initscript name=\"" + hashrspec[clef]["name"]  + "\"/>" +
 237                         "  </sliver_type>")
 238         for infohost in itemshost:
 239             bonfires.append("  <bonfire:attribute name=\"" + infohost + "\"value=\"" + hashrspec[clef][infohost]  + "\"/>")
 240         bonfires.append("</node>")
 241
 242 # remove the useless xml tag version
 243 def remove_needless_txt(txt):
 244     txt=str(txt)
 245     txt=txt.replace("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n","\n")
 246     txt=txt.replace("<?xml version='1.0' encoding='UTF-8'?>\n","\n")
 247     return txt
 248
 249 # list all bonfire resources following the sfa specification in a rspec way
 250 def bonsources():
 251         # parameters
 252     locations = ["fr-inria", "be-ibbt", "uk-epcc"]
 253     urlnetworks = buildpagehttp("https://api.integration.bonfire.grid5000.fr/locations/", "networks", locations)
 254     urlstorages = buildpagehttp("https://api.integration.bonfire.grid5000.fr/locations/", "storages", locations)
 255     urlcomputes = buildpagehttp("https://api.integration.bonfire.grid5000.fr/locations/", "computes", locations)
 256     # main code
 257     bonfires = []
 258     generatedtime =  time.strftime("%FT%T%Z")
 259     sfabegin = "<RSpec type=\"SFA\" generated=" + generatedtime + "\">"
 260     bonfires.append("<?xml version=\"1.0\"?>")
 261     bonfires.append(sfabegin)
 262     bonfires.append("<managed_experiments>")
 263     manag_exp =  remove_needless_txt(callcurl("https://api.bonfire-project.eu/managed_experiments"))
 264     bonfires.append(manag_exp)
 265     bonfires.append("</managed_experiments><sites><machines>")
 266     jfedfeat(bonfires, "http://frontend.bonfire.grid5000.fr/one-status.xml")
 267     jfedfeat(bonfires, "http://bonfire.epcc.ed.ac.uk/one-status.xml")
 268     jfedfeat(bonfires, "http://bonfire.psnc.pl/one-status.xml")
 269     jfedfeat(bonfires, "http://nebulosus.rus.uni-stuttgart.de/one-status.xml")
 270     bonfires.append("</machines><networks>")
 271     # adding networks information
 272     for xmlnetworks in urlnetworks:
 273         bonfires.append(remove_needless_txt(callcurl(xmlnetworks)))
 274     bonfires.append("</networks><storages>")
 275     # adding storages information
 276     for xmlstorages in urlstorages:
 277         bonfires.append(remove_needless_txt(callcurl(xmlstorages)))
 278     bonfires.append("</storages><instance_types><computes>")
 279     # adding computes information
 280     for xmlcomputes in urlcomputes:
 281         bonfires.append(remove_needless_txt(callcurl(xmlcomputes)))
 282     bonfires.append("</computes></instance_types></sites><experiments>")
 283     exp = callcurl("https://api.integration.bonfire.grid5000.fr/experiments")
 284     rexp = remove_needless_txt(exp)
 285     bonfires.append(rexp)
 286     bonfires.append("</experiments><reservations>")
 287     # adding reservation information
 288     reserv = callcurl("https://api.integration.bonfire.grid5000.fr/locations/fr-inria/reservations")
 289     rreserv = remove_needless_txt(reserv)
 290     bonfires.append(rreserv)
 291     bonfires.append("</reservations>")
 292     bonfires.append("</RSpec>")
 293     bonfires = "\n".join(bonfires)
 294     bonfires = bonfires.replace("\n\n","")
 295     return bonfires