sfa/bonfire/bonfire.py

   1 #!/usr/bin/python
   2 # -*- coding:utf-8 -*-
   3 #yum -y install python-pip
   4 #pip install requests
   5 import requests
   6 import xml.etree.ElementTree as ET
   7 import subprocess
   8 import time
   9
  10 # module for bonfire to connect with sfa (following the Rspec)
  11 # inspired by the following documenation :
  12 # https://svn.planet-lab.org/wiki/SfaDeveloperDummyTutorial#RunningSFAinDummyflavour
  13
  14 # 1) list all the resources  of bonfire from sfa's point of view
  15 # python -c 'import bonfire; print bonfire.bonsources()'
  16
  17 # 2) retrieve the url, the name and the key that will currently use by sfa for a compute N°3656 located at fr-inria
  18 # python -c 'import bonfire; print bonfire.rsa_user_bonfire("fr-inria", "3656")'
  19
  20 # 3) create a new user and slice for sfa wrap
  21 # python -c 'import bonfire; print bonfire.new_user_slice()'
  22
  23 # 4) changing the status to running status for the experiment 2911
  24 # python -c 'import bonfire; print bonfire.provisioning("2911")'
  25
  26 # 5) stop virtual machine n°3756  at fr-inira testbed
  27 # python -c 'import bonfire; print bonfire.stop_vm("fr-inria", "3756")'
  28
  29 # 6) allocation : create an experiment bonfire with slice information
  30 # python -c 'import bonfire; print bonfire.allocate("nlebreto", "nlebreto", "tdes", "125", "topdomain.dummy.nicolasi", "https://api.integration.bonfire.grid5000.fr/experiments")'
  31
  32 # 7) remove slice or key
  33 # python -c 'import bonfire; print bonfire.remove_slice("topdomain.dummy.alice_slice")'
  34
  35 # 8) attach slice to a user (did not work)
  36 # python -c 'import bonfire; print bonfire.create_slice_attach_user("topdomain.dummy.alice")'
  37
  38 # 9) verify bonfire authentication
  39 # # python -c 'import bonfire; print bonfire.callcurl("https://api.bonfire-project.eu/")'
  40
  41 # ########################################################## #
  42 # ########################################################## #
  43
  44 # pseudo authentication for bonfire
  45 def bonfire_authenticate():
  46     h = {}
  47     h["user"]      = "nlebreto"
  48     h["user_pass"] = "GDRU_23tc$"
  49     h["location"]  = "https://api.integration.bonfire.grid5000.fr"
  50     return h
  51
  52 # create a slice and attach a specific user to it
  53 def create_slice_attach_user(user_slice):
  54     call = "sfa.py add -x {0}_slice -t slice -r {0}@dummy.net".format(user_slice)
  55     callcreateslice =  subprocess.Popen(call, shell=True)
  56
  57 # remove slice or key
  58 def remove_slice(name):
  59     cmdremove    = "sfaadmin.py reg remove {0}".format(name)
  60     removeaction = subprocess.Popen(cmdremove, shell=True)
  61
  62 # show specific credential of a slice
  63 def show_slice_credential(slice_name):
  64     path = "/root/.sfi/{0}.slice.cred".format(slice_name)
  65     tree = ET.parse(path)
  66     root = tree.getroot()
  67     hash = {}
  68     hash["slice_native"] = root.findall(".//signatures//{http://www.w3.org/2000/09/xmldsig#}Signature//{http://www.w3.org/2000/09/xmldsig#}KeyInfo//{http://www.w3.org/2000/09/xmldsig#}X509Data//{http://www.w3.org/2000/09/xmldsig#}X509SubjectName")[0].text
  69     for target in root.findall('credential'):
  70         hash["slice_user_urn"] = target.find('owner_urn').text
  71         hash["slice_urn"] = target.find('target_urn').text
  72         hash["serial"] = target.find('serial').text
  73     return hash
  74
  75 # create a bonfire experiment from a sfa point of view
  76 def allocate(user_name, groups, description, walltime, slice_name):
  77     hash ={}
  78     hash = show_slice_credential(slice_name)
  79     create_fed4fire_exp(user_name, groups, description, walltime, hash["slice_urn"], hash["slice_user_urn"], hash["slice_native"])
  80
  81
  82
  83 # retrieve the url, the name and the key that will currently use by sfa
  84 def rsa_user_bonfire(testbed, num_compute):
  85     url = "https://api.integration.bonfire.grid5000.fr/" + "locations/" + testbed + "/computes/" + num_compute
  86     pagebonfirecompute = callcurl(url)
  87     xmlreduit = ET.fromstring(pagebonfirecompute)
  88     hash = {}
  89     hash["url"] = url
  90     for name in xmlreduit:
  91         if name.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}groups":
  92            hash["name"] = name.text
  93         for context in name:
  94             if context.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}authorized_keys":
  95                hash["keys"] = context.text
  96     return hash
  97
  98 # create a new user and slice for sfa wrap
  99 def new_user_slice():
 100     n = rsa_user_bonfire("fr-inria", "3656")
 101     #url = n["url"] + "." + n["name"]
 102     # fix to do add -k id_rsa.pub (pb key convert)
 103     url = "topdomain.dummy." + n["name"]
 104     txtcreateuser = "sfaadmin.py reg register -x {0} -t user -e {1}@dummy.net".format(url, n["name"])
 105     createusersfa = subprocess.Popen(txtcreateuser, shell=True)
 106     #slice = n["url"] + "." + n["name"] + "_" + n["name"]
 107     slice = "topdomain.dummy." + n["name"] + "_slice"
 108     txtslice = "sfaadmin.py reg register -x {0} -t slice -r {1}".format(slice, url)
 109     createslice = subprocess.Popen(txtslice, shell=True)
 110
 111 # create a experiment bonfire with the slice urn and the experiment owner
 112 def create_fed4fire_exp(name, groups, description, walltime, slice_urn, slice_user_urn, slice_native, url_experiment_bonfire):
 113     xmldescription='<experiment xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><name>' + name +'</name><groups>' + groups + '</groups><description>' + description + '</description><walltime>' + walltime + '</walltime><status>ready</status><slice_urn>' + slice_urn + '</slice_urn><slice_usr_urn>' + slice_user_urn + '<slice_usr_urn><slice_native>' + slice_native + '</slice_native></experiment>'
 114     postexp(url_experiment_bonfire, xmldescription)
 115
 116 # simple post method for request
 117 def postexp(url, xmldescription):
 118     headers = {'content-type': 'application/vnd.bonfire+xml'}
 119     h = bonfire_authenticate()
 120     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=(h["user"], h["user_pass"]))
 121
 122 # stop a virtual machine for bonfire
 123 # changing the state to stopped state
 124 def stop_vm(testbed, num_compute):
 125     url = "https://api.integration.bonfire.grid5000.fr/" + "locations/" + testbed + "/computes/" + num_compute
 126     xmldescription = '<compute xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><state>stopped</state></compute>'
 127     headers = {'content-type': 'application/vnd.bonfire+xml'}
 128     h = bonfire_authenticate()
 129     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=(h["user"], h["user_pass"]))
 130
 131 # provisioning : set a bonfire's experiment to running
 132 # changing the status to running status
 133 def provisioning(num_experiment):
 134     url = "https://api.integration.bonfire.grid5000.fr/experiments/" + num_experiment
 135     xmldescription = '<experiment xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><status>running</status></experiment>'
 136     headers = {'content-type': 'application/vnd.bonfire+xml'}
 137     h = bonfire_authenticate()
 138     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=(h["user"], h["user_pass"]))
 139
 140 # retrieving the url, the name and the keys for a specific compute
 141 def rsa_user_bonfire(testbed, num_compute):
 142     url = "https://api.integration.bonfire.grid5000.fr/" + "locations/" + testbed + "/computes/" + num_compute
 143     pagebonfirecompute = callcurl(url)
 144     xmlreduit = ET.fromstring(pagebonfirecompute)
 145     hash = {}
 146     hash["url"] = url
 147     for name in xmlreduit:
 148         if name.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}groups":
 149            hash["name"] = name.text
 150         for context in name:
 151             if context.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}authorized_keys":
 152                hash["keys"] = context.text
 153     return hash
 154
 155 # do a curl request
 156 def callcurl(url):
 157     h = bonfire_authenticate()
 158     r = requests.get(url, verify=False, auth=(h["user"], h["user_pass"]))
 159     if r.status_code == 401:
 160         return "error 401, you need to be register to the portal f4f"
 161     if r.status_code == 200:
 162         return r.text
 163
 164 # create the url page
 165 def buildpagehttp(part1, part2, locations):
 166     res = []
 167     for page in locations:
 168         res.append(part1 + page  + "/" + part2)
 169     return res
 170
 171 def boucle(itemname, xmltree, hashrspec, name):
 172     for item in xmltree.findall(itemname):
 173         hashrspec[name.text][itemname] = item.text
 174
 175 # method to list all information from testbeds
 176 def jfedfeat(bonfires, pageurl):
 177     pageforstatus = callcurl(pageurl)
 178     xmlreduit = ET.fromstring(pageforstatus)
 179     hashrspec = {}
 180     itemshost = ["DISK_USAGE", "MEM_USAGE", "CPU_USAGE", "MAX_DISK", "MAX_MEM",  "MAX_CPU",
 181                  "FREE_DISK",  "FREE_MEM",  "FREE_CPU", "FREE_MEM",  "FREE_CPU", "USED_DISK",
 182                  "USED_MEM",   "USED_CPU",  "RUNNING_VMS"
 183                 ]
 184     # retrieve info for xml tree
 185     for host in xmlreduit.findall('HOST'):
 186         for name in host.findall('NAME'):
 187             hashrspec[name.text] = {"name" : name.text}
 188             for hostshare in host.findall('HOST_SHARE'):
 189                 for itemshostname in itemshost:
 190                     boucle(itemshostname, hostshare, hashrspec, name)
 191
 192  # jfed feature
 193     for clef in hashrspec:
 194         bonfires.append("<node component_manager_id=\"urn:publicid:IDN+topdomain+authority+cm" +
 195                         " component_id=\"urn:publicid:IDN+topdomain:" + hashrspec[clef]["name"] +
 196                         "\" component_name=" + hashrspec[clef]["name"] + "exclusive=\"false\">" +
 197                         "  <location country=\"unknown\" longitude=\"123456\" latitude=\"654321\"/>" +
 198                         "  <interface component_id=\"urn:publicid:IDN+ple+interface+node14312:eth0\"/>" +
 199                         "  <available now=\"true\"/>" +
 200                         "  <sliver_type name=\"" + hashrspec[clef]["name"] + "\">" +
 201                         "      <bonfire:initscript name=\"" + hashrspec[clef]["name"]  + "\"/>" +
 202                         "  </sliver_type>")
 203         for infohost in itemshost:
 204             bonfires.append("  <bonfire:attribute name=\"" + infohost + "\"value=\"" + hashrspec[clef][infohost]  + "\"/>")
 205         bonfires.append("</node>")
 206
 207 # remove the useless xml tag version
 208 def remove_needless_txt(txt):
 209     txt=str(txt)
 210     txt=txt.replace("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n","\n")
 211     txt=txt.replace("<?xml version='1.0' encoding='UTF-8'?>\n","\n")
 212     return txt
 213
 214 # list all bonfire resources following the sfa specification
 215 def bonsources():
 216         # parameters
 217     locations = ["fr-inria", "be-ibbt", "uk-epcc"]
 218     urlnetworks = buildpagehttp("https://api.integration.bonfire.grid5000.fr/locations/", "networks", locations)
 219     urlstorages = buildpagehttp("https://api.integration.bonfire.grid5000.fr/locations/", "storages", locations)
 220     urlcomputes = buildpagehttp("https://api.integration.bonfire.grid5000.fr/locations/", "computes", locations)
 221     # main code
 222     bonfires = []
 223     generatedtime =  time.strftime("%FT%T%Z")
 224     sfabegin = "<RSpec type=\"SFA\" generated=" + generatedtime + "\">"
 225     bonfires.append("<?xml version=\"1.0\"?>")
 226     bonfires.append(sfabegin)
 227     bonfires.append("<managed_experiments>")
 228     manag_exp =  remove_needless_txt(callcurl("https://api.bonfire-project.eu/managed_experiments"))
 229     bonfires.append(manag_exp)
 230     bonfires.append("</managed_experiments><sites><machines>")
 231     jfedfeat(bonfires, "http://frontend.bonfire.grid5000.fr/one-status.xml")
 232     jfedfeat(bonfires, "http://bonfire.epcc.ed.ac.uk/one-status.xml")
 233     jfedfeat(bonfires, "http://bonfire.psnc.pl/one-status.xml")
 234     jfedfeat(bonfires, "http://nebulosus.rus.uni-stuttgart.de/one-status.xml")
 235     bonfires.append("</machines><networks>")
 236     for xmlnetworks in urlnetworks:
 237         bonfires.append(remove_needless_txt(callcurl(xmlnetworks)))
 238     bonfires.append("</networks><storages>")
 239     for xmlstorages in urlstorages:
 240         bonfires.append(remove_needless_txt(callcurl(xmlstorages)))
 241     bonfires.append("</storages><instance_types><computes>")
 242     for xmlcomputes in urlcomputes:
 243         bonfires.append(remove_needless_txt(callcurl(xmlcomputes)))
 244     bonfires.append("</computes></instance_types></sites><experiments>")
 245     exp = callcurl("https://api.integration.bonfire.grid5000.fr/experiments")
 246     rexp = remove_needless_txt(exp)
 247     bonfires.append(rexp)
 248     bonfires.append("</experiments><reservations>")
 249     reserv = callcurl("https://api.integration.bonfire.grid5000.fr/locations/fr-inria/reservations")
 250     rreserv = remove_needless_txt(reserv)
 251     bonfires.append(rreserv)
 252     bonfires.append("</reservations>")
 253     bonfires.append("</RSpec>")
 254     bonfires = "\n".join(bonfires)
 255     bonfires = bonfires.replace("\n\n","")
 256     return bonfires