sfa/bonfire/bonfire.py

   1 #!/usr/bin/python
   2 # -*- coding:utf-8 -*-
   3 #yum -y install python-pip
   4 #pip install requests
   5 import requests
   6 import xml.etree.ElementTree as ET
   7 import subprocess
   8 import time
   9
  10 # module for bonfire to connect with sfa (following the Rspec)
  11 # inspired by the following documenation :
  12 # https://svn.planet-lab.org/wiki/SfaDeveloperDummyTutorial#RunningSFAinDummyflavour
  13
  14 # 1) list all the resources from bonfire's point of view
  15 # python -c 'import bonfire; print bonfire.bonsources()'
  16
  17 # 2) retrieve the url, the name and the key that will currently use by sfa for a compute N°3656 located at fr-inria
  18 # python -c 'import bonfire; print bonfire.rsa_user_bonfire("fr-inria", "3656")
  19
  20 # 3) create a new user and slice for sfa wrap
  21 # python -c 'import bonfire; print bonfire.new_user_slice()
  22
  23 # 4) changing the status to running status for the experiment 2911
  24 # python -c 'import bonfire; print bonfire.provisioning("2911")
  25
  26 # 5) stop virtual machine n°3756  at fr-inira testbed
  27 # python -c 'import bonfire; print bonfire.stop_vm("fr-inria", "3756")
  28
  29 # 6) create an experiment bonfire with the slice id n°2345423 and the experiment owner : foo+bar.stuff+emulab+professor
  30 # python -c 'import bonfire; print bonfire.create_fed4fire_exp("nlebreto", "nlebreto", "desc", "300", "2345423", "foo+bar.stuff+emulab+professor")
  31
  32
  33
  34 # ########################################################## #
  35
  36
  37 # retrieve the url, the name and the key that will currently use by sfa
  38 def rsa_user_bonfire(testbed, num_compute):
  39     url = "https://api.integration.bonfire.grid5000.fr/" + "locations/" + testbed + "/computes/" + num_compute
  40     pagebonfirecompute = callcurl(url)
  41     xmlreduit = ET.fromstring(pagebonfirecompute)
  42     hash = {}
  43     hash["url"] = url
  44     for name in xmlreduit:
  45         if name.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}groups":
  46            hash["name"] = name.text
  47         for context in name:
  48             if context.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}authorized_keys":
  49                hash["keys"] = context.text
  50     return hash
  51
  52 # create a new user and slice for sfa wrap
  53 def new_user_slice():
  54     n = rsa_user_bonfire("fr-inria", "3656")
  55     #url = n["url"] + "." + n["name"]
  56     # fix to do add -k id_rsa.pub (pb key convert)
  57     url = "topdomain.dummy." + n["name"]
  58     txtcreateuser = "sfaadmin.py reg register -x {0} -t user -e {1}@dummy.net".format(url, n["name"])
  59     createusersfa = subprocess.Popen(txtcreateuser, shell=True)
  60     #slice = n["url"] + "." + n["name"] + "_" + n["name"]
  61     slice = "topdomain.dummy." + n["name"] + "_slice"
  62     txtslice = "sfaadmin.py reg register -x {0} -t slice -r {1}".format(slice, url)
  63     createslice = subprocess.Popen(txtslice, shell=True)
  64
  65 # create a experiment bonfire with the slice id and the experiment owner
  66 def create_fed4fire_exp(name, groups, description, walltime, slice_id, exp_owner):
  67    # fix this to have the exact mapping to retrieve slice
  68    # xmldescription='<experiment xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><name>' + name +'</name><groups>' + groups + '</groups><description>' + description + '</description><walltime>' + walltime + '</walltime><status>ready</status><fed4fire><slice_id>' + slice_id + '</slice_id><exp_owner>' + exp_owner + '<exp_owner></fed4fire></experiment>'
  69     postexp("https://api.integration.bonfire.grid5000.fr/experiments", '<experiment xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><name>testnlebreto</name><groups>nlebreto</groups><description>sdg</description><walltime>3000</walltime><status>ready</status><fed4fire><slice_id>2345423</slice_id><exp_owner>foo+bar.stuff+emulab+professor<exp_owner></fed4fire></experiment>')
  70
  71 # simple post method for request
  72 def postexp(url, xmldescription):
  73     headers = {'content-type': 'application/vnd.bonfire+xml'}
  74     r = requests.post(url, data=xmldescription, headers=headers, verify=False, auth=('nlebreto', 'GDRU_23tc$'))
  75
  76 # stop a virtual machine for bonfire
  77 # changing the state to stopped state
  78 def stop_vm(testbed, num_compute):
  79     url = "https://api.integration.bonfire.grid5000.fr/" + "locations/" + testbed + "/computes/" + num_compute
  80     xmldescription = '<compute xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><state>stopped</state></compute>'
  81     headers = {'content-type': 'application/vnd.bonfire+xml'}
  82     requests.put(url, data=xmldescription, headers=headers, verify=False, auth=('nlebreto', 'GDRU_23tc$'))
  83
  84 # provisioning : set an experiment to running
  85 # changing the status to running status
  86 def provisioning(num_experiment):
  87     url = "https://api.integration.bonfire.grid5000.fr/experiments/" + num_experiment
  88     xmldescription = '<experiment xmlns="http://api.bonfire-project.eu/doc/schemas/occi"><status>running</status></experiment>'
  89     headers = {'content-type': 'application/vnd.bonfire+xml'}
  90     requests.put(url, data=xmldescription, headers=headers, verify=False, auth=('nlebreto', 'GDRU_23tc$'))
  91
  92 # retrieving the url, the name and the keys for a specific compute
  93 def rsa_user_bonfire(testbed, num_compute):
  94     url = "https://api.integration.bonfire.grid5000.fr/" + "locations/" + testbed + "/computes/" + num_compute
  95     pagebonfirecompute = callcurl(url)
  96     xmlreduit = ET.fromstring(pagebonfirecompute)
  97     hash = {}
  98     hash["url"] = url
  99     for name in xmlreduit:
 100         if name.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}groups":
 101            hash["name"] = name.text
 102         for context in name:
 103             if context.tag == "{http://api.bonfire-project.eu/doc/schemas/occi}authorized_keys":
 104                hash["keys"] = context.text
 105     return hash
 106
 107 # do a curl request
 108 def callcurl(url):
 109     r = requests.get(url, verify=False, auth=('nlebreto', 'GDRU_23tc$'))
 110     if r.status_code == 200:
 111         return r.text
 112
 113 # create the url page
 114 def buildpagehttp(part1, part2, locations):
 115     res = []
 116     for page in locations:
 117         res.append(part1 + page  + "/" + part2)
 118     return res
 119
 120 def boucle(itemname, xmltree, hashrspec, name):
 121     for item in xmltree.findall(itemname):
 122         hashrspec[name.text][itemname] = item.text
 123
 124 # method to list all information from testbeds
 125 def jfedfeat(bonfires, pageurl):
 126     pageforstatus = callcurl(pageurl)
 127     xmlreduit = ET.fromstring(pageforstatus)
 128     hashrspec = {}
 129     itemshost = ["DISK_USAGE", "MEM_USAGE", "CPU_USAGE", "MAX_DISK", "MAX_MEM",  "MAX_CPU",
 130                  "FREE_DISK",  "FREE_MEM",  "FREE_CPU", "FREE_MEM",  "FREE_CPU", "USED_DISK",
 131                  "USED_MEM",   "USED_CPU",  "RUNNING_VMS"
 132                 ]
 133     # retrieve info for xml tree
 134     for host in xmlreduit.findall('HOST'):
 135         for name in host.findall('NAME'):
 136             hashrspec[name.text] = {"name" : name.text}
 137             for hostshare in host.findall('HOST_SHARE'):
 138                 for itemshostname in itemshost:
 139                     boucle(itemshostname, hostshare, hashrspec, name)
 140
 141  # jfed feature
 142     for clef in hashrspec:
 143         bonfires.append("<node component_manager_id=\"urn:publicid:IDN+topdomain+authority+cm" +
 144                         " component_id=\"urn:publicid:IDN+topdomain:" + hashrspec[clef]["name"] +
 145                         "\" component_name=" + hashrspec[clef]["name"] + "exclusive=\"false\">" +
 146                         "  <location country=\"unknown\" longitude=\"123456\" latitude=\"654321\"/>" +
 147                         "  <interface component_id=\"urn:publicid:IDN+ple+interface+node14312:eth0\"/>" +
 148                         "  <available now=\"true\"/>" +
 149                         "  <sliver_type name=\"" + hashrspec[clef]["name"] + "\">" +
 150                         "      <bonfire:initscript name=\"" + hashrspec[clef]["name"]  + "\"/>" +
 151                         "  </sliver_type>")
 152         for infohost in itemshost:
 153             bonfires.append("  <bonfire:attribute name=\"" + infohost + "\"value=\"" + hashrspec[clef][infohost]  + "\"/>")
 154         bonfires.append("</node>")
 155
 156 # remove the useless xml tag version
 157 def remove_needless_txt(txt):
 158     txt=str(txt)
 159     txt=txt.replace("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n","\n")
 160     txt=txt.replace("<?xml version='1.0' encoding='UTF-8'?>\n","\n")
 161     return txt
 162
 163 # list all bonfire resources following the sfa specification
 164 def bonsources():
 165         # parameters
 166     locations = ["fr-inria", "be-ibbt", "uk-epcc"]
 167     urlnetworks = buildpagehttp("https://api.integration.bonfire.grid5000.fr/locations/", "networks", locations)
 168     urlstorages = buildpagehttp("https://api.integration.bonfire.grid5000.fr/locations/", "storages", locations)
 169     urlcomputes = buildpagehttp("https://api.integration.bonfire.grid5000.fr/locations/", "computes", locations)
 170     # main code
 171     bonfires = []
 172     generatedtime =  time.strftime("%FT%T%Z")
 173     sfabegin = "<RSpec type=\"SFA\" generated=" + generatedtime + "\">"
 174     bonfires.append("<?xml version=\"1.0\"?>")
 175     bonfires.append(sfabegin)
 176     bonfires.append("<managed_experiments>")
 177     manag_exp =  remove_needless_txt(callcurl("https://api.bonfire-project.eu/managed_experiments"))
 178     bonfires.append(manag_exp)
 179     bonfires.append("</managed_experiments><sites><machines>")
 180     jfedfeat(bonfires, "http://frontend.bonfire.grid5000.fr/one-status.xml")
 181     jfedfeat(bonfires, "http://bonfire.epcc.ed.ac.uk/one-status.xml")
 182     jfedfeat(bonfires, "http://bonfire.psnc.pl/one-status.xml")
 183     jfedfeat(bonfires, "http://nebulosus.rus.uni-stuttgart.de/one-status.xml")
 184     bonfires.append("</machines><networks>")
 185     for xmlnetworks in urlnetworks:
 186         bonfires.append(remove_needless_txt(callcurl(xmlnetworks)))
 187     bonfires.append("</networks><storages>")
 188     for xmlstorages in urlstorages:
 189         bonfires.append(remove_needless_txt(callcurl(xmlstorages)))
 190     bonfires.append("</storages><instance_types><computes>")
 191     for xmlcomputes in urlcomputes:
 192         bonfires.append(remove_needless_txt(callcurl(xmlcomputes)))
 193     bonfires.append("</computes></instance_types></sites><experiments>")
 194     exp = callcurl("https://api.integration.bonfire.grid5000.fr/experiments")
 195     rexp = remove_needless_txt(exp)
 196     bonfires.append(rexp)
 197     bonfires.append("</experiments><reservations>")
 198     reserv = callcurl("https://api.integration.bonfire.grid5000.fr/locations/fr-inria/reservations")
 199     rreserv = remove_needless_txt(reserv)
 200     bonfires.append(rreserv)
 201     bonfires.append("</reservations>")
 202     bonfires.append("</RSpec>")
 203     bonfires = "\n".join(bonfires)
 204     bonfires = bonfires.replace("\n\n","")
 205     return bonfires