1 from __future__ import with_statement
7 from boto.ec2.regioninfo import RegionInfo
8 from boto.exception import EC2ResponseError
9 from ConfigParser import ConfigParser
10 from xmlbuilder import XMLBuilder
11 from lxml import etree as ET
12 from sqlobject import *
14 from sfa.util.faults import *
15 from sfa.util.xrn import urn_to_hrn
16 from sfa.util.rspec import RSpec
17 from sfa.server.registry import Registries
18 from sfa.trust.credential import Credential
19 from sfa.plc.api import SfaAPI
20 from sfa.util.plxrn import hrn_to_pl_slicename, slicename_to_hrn
23 # The data structure used to represent a cloud.
24 # It contains the cloud name, its ip address, image information,
25 # key pairs, and clusters information.
30 # The location of the RelaxNG schema.
32 EUCALYPTUS_RSPEC_SCHEMA='/etc/sfa/eucalyptus.rng'
35 sys.stderr = file('/var/log/euca_agg.log', 'a+')
39 # A representation of an Eucalyptus instance. This is a support class
40 # for instance <-> slice mapping.
42 class EucaInstance(SQLObject):
43 instance_id = StringCol(unique=True, default=None)
44 kernel_id = StringCol()
45 image_id = StringCol()
46 ramdisk_id = StringCol()
47 inst_type = StringCol()
48 key_pair = StringCol()
49 slice = ForeignKey('Slice')
52 # Contacts Eucalyptus and tries to reserve this instance.
54 # @param botoConn A connection to Eucalyptus.
55 # @param pubKeys A list of public keys for the instance.
57 def reserveInstance(self, botoConn, pubKeys):
58 print >>sys.stderr, 'Reserving an instance: image: %s, kernel: ' \
59 '%s, ramdisk: %s, type: %s, key: %s' % \
60 (self.image_id, self.kernel_id, self.ramdisk_id,
61 self.inst_type, self.key_pair)
63 # XXX The return statement is for testing. REMOVE in production
67 reservation = botoConn.run_instances(self.image_id,
68 kernel_id = self.kernel_id,
69 ramdisk_id = self.ramdisk_id,
70 instance_type = self.inst_type,
71 key_name = self.key_pair,
73 for instance in reservation.instances:
74 self.instance_id = instance.id
76 # If there is an error, destroy itself.
77 except EC2ResponseError, ec2RespErr:
78 errTree = ET.fromstring(ec2RespErr.body)
79 msg = errTree.find('.//Message')
80 print >>sys.stderr, msg.text
84 # A representation of a PlanetLab slice. This is a support class
85 # for instance <-> slice mapping.
87 class Slice(SQLObject):
88 slice_hrn = StringCol()
89 #slice_index = DatabaseIndex('slice_hrn')
90 instances = MultipleJoin('EucaInstance')
93 # Initialize the aggregate manager by reading a configuration file.
96 configParser = ConfigParser()
97 configParser.read(['/etc/sfa/eucalyptus_aggregate.conf', 'eucalyptus_aggregate.conf'])
98 if len(configParser.sections()) < 1:
99 print >>sys.stderr, 'No cloud defined in the config file'
100 raise Exception('Cannot find cloud definition in configuration file.')
102 # Only read the first section.
103 cloudSec = configParser.sections()[0]
104 cloud['name'] = cloudSec
105 cloud['access_key'] = configParser.get(cloudSec, 'access_key')
106 cloud['secret_key'] = configParser.get(cloudSec, 'secret_key')
107 cloud['cloud_url'] = configParser.get(cloudSec, 'cloud_url')
108 cloudURL = cloud['cloud_url']
109 if cloudURL.find('https://') >= 0:
110 cloudURL = cloudURL.replace('https://', '')
111 elif cloudURL.find('http://') >= 0:
112 cloudURL = cloudURL.replace('http://', '')
113 (cloud['ip'], parts) = cloudURL.split(':')
115 # Read the bundle images config file.
116 if not os.path.exists('/etc/sfa/bundle_image.conf') and \
117 not os.path.exists('bundle_image.conf'):
118 print >>sys.stderr, 'Could not find bundle_image.conf'
119 raise Exception('Could not find bundle_image.conf')
120 imageBundleParser = ConfigParser()
121 imageBundleParser.read(['/etc/sfa/bundle_image.conf', 'bundle_image.conf'])
122 cloud['imageBundles'] = {}
123 for bundle in imageBundleParser.sections():
124 info = dict(imageBundleParser.items(bundle))
125 cloud['imageBundles'][bundle] = info
127 # Initialize sqlite3 database.
128 dbPath = '/etc/sfa/db'
129 dbName = 'euca_aggregate.db'
131 if not os.path.isdir(dbPath):
132 print >>sys.stderr, '%s not found. Creating directory ...' % dbPath
135 conn = connectionForURI('sqlite://%s/%s' % (dbPath, dbName))
136 sqlhub.processConnection = conn
137 Slice.createTable(ifNotExists=True)
138 EucaInstance.createTable(ifNotExists=True)
140 # Make sure the schema exists.
141 if not os.path.exists(EUCALYPTUS_RSPEC_SCHEMA):
142 err = 'Cannot location schema at %s' % EUCALYPTUS_RSPEC_SCHEMA
143 print >>sys.stderr, err
147 # Creates a connection to Eucalytpus. This function is inspired by
148 # the make_connection() in Euca2ools.
150 # @return A connection object or None
152 def getEucaConnection():
154 accessKey = cloud['access_key']
155 secretKey = cloud['secret_key']
156 eucaURL = cloud['cloud_url']
161 if not accessKey or not secretKey or not eucaURL:
162 print >>sys.stderr, 'Please set ALL of the required environment ' \
163 'variables by sourcing the eucarc file.'
166 # Split the url into parts
167 if eucaURL.find('https://') >= 0:
169 eucaURL = eucaURL.replace('https://', '')
170 elif eucaURL.find('http://') >= 0:
172 eucaURL = eucaURL.replace('http://', '')
173 (eucaHost, parts) = eucaURL.split(':')
175 parts = parts.split('/')
176 eucaPort = int(parts[0])
178 srvPath = '/'.join(parts)
180 return boto.connect_ec2(aws_access_key_id=accessKey,
181 aws_secret_access_key=secretKey,
183 region=RegionInfo(None, 'eucalyptus', eucaHost),
188 # Returns a string of keys that belong to the users of the given slice.
189 # @param sliceHRN The hunman readable name of the slice.
192 def getKeysForSlice(sliceHRN):
194 # convert hrn to slice name
195 plSliceName = hrn_to_pl_slicename(sliceHRN)
196 except IndexError, e:
197 print >>sys.stderr, 'Invalid slice name (%s)' % sliceHRN
200 # Get the slice's information
201 sliceData = api.plshell.GetSlices(api.plauth, {'name':plSliceName})
203 print >>sys.stderr, 'Cannot get any data for slice %s' % plSliceName
206 # It should only return a list with len = 1
207 sliceData = sliceData[0]
210 person_ids = sliceData['person_ids']
212 print >>sys.stderr, 'No users in slice %s' % sliceHRN
215 persons = api.plshell.GetPersons(api.plauth, person_ids)
216 for person in persons:
217 pkeys = api.plshell.GetKeys(api.plauth, person['key_ids'])
219 keys.append(key['key'])
224 # A class that builds the RSpec for Eucalyptus.
226 class EucaRSpecBuilder(object):
228 # Initizes a RSpec builder
230 # @param cloud A dictionary containing data about a
231 # cloud (ex. clusters, ip)
232 def __init__(self, cloud):
233 self.eucaRSpec = XMLBuilder(format = True, tab_step = " ")
234 self.cloudInfo = cloud
237 # Creates a request stanza.
239 # @param num The number of instances to create.
240 # @param image The disk image id.
241 # @param kernel The kernel image id.
242 # @param keypair Key pair to embed.
243 # @param ramdisk Ramdisk id (optional).
245 def __requestXML(self, num, image, kernel, keypair, ramdisk = ''):
250 with xml.kernel_image(id=kernel):
256 with xml.ramdisk(id=ramdisk):
258 with xml.disk_image(id=image):
264 # Creates the cluster stanza.
266 # @param clusters Clusters information.
268 def __clustersXML(self, clusters):
269 cloud = self.cloudInfo
272 for cluster in clusters:
273 instances = cluster['instances']
274 with xml.cluster(id=cluster['name']):
278 for inst in instances:
279 with xml.vm_type(name=inst[0]):
282 with xml.max_instances:
286 with xml.memory(unit='MB'):
288 with xml.disk_space(unit='GB'):
290 if 'instances' in cloud and inst[0] in cloud['instances']:
291 existingEucaInstances = cloud['instances'][inst[0]]
292 with xml.euca_instances:
293 for eucaInst in existingEucaInstances:
294 with xml.euca_instance(id=eucaInst['id']):
296 xml << eucaInst['state']
298 xml << eucaInst['public_dns']
300 xml << eucaInst['key']
302 def __imageBundleXML(self, bundles):
305 for bundle in bundles.keys():
306 print >>sys.stderr, 'bundle: %r' % bundle
308 with xml.bundle(id=bundle):
309 with xml.description:
310 xml << bundles[bundle]['description']
313 # Creates the Images stanza.
315 # @param images A list of images in Eucalyptus.
317 def __imagesXML(self, images):
321 with xml.image(id=image.id):
325 xml << image.architecture
329 xml << image.location
332 # Creates the KeyPairs stanza.
334 # @param keypairs A list of key pairs in Eucalyptus.
336 def __keyPairsXML(self, keypairs):
344 # Generates the RSpec.
347 if not self.cloudInfo:
348 print >>sys.stderr, 'No cloud information'
352 cloud = self.cloudInfo
353 with xml.RSpec(type='eucalyptus'):
354 with xml.cloud(id=cloud['name']):
357 self.__keyPairsXML(cloud['keypairs'])
358 self.__imagesXML(cloud['images'])
359 self.__imageBundleXML(cloud['imageBundles'])
360 self.__clustersXML(cloud['clusters'])
364 # A parser to parse the output of availability-zones.
366 # Note: Only one cluster is supported. If more than one, this will
369 class ZoneResultParser(object):
370 def __init__(self, zones):
374 if len(self.zones) < 3:
380 cluster['name'] = self.zones[0].name
381 cluster['ip'] = self.zones[0].state
383 for i in range(2, len(self.zones)):
384 currZone = self.zones[i]
385 instType = currZone.name.split()[1]
387 stateString = currZone.state.split('/')
388 rscString = stateString[1].split()
390 instFree = int(stateString[0])
391 instMax = int(rscString[0])
392 instNumCpu = int(rscString[1])
393 instRam = int(rscString[2])
394 instDiskSpace = int(rscString[3])
396 instTuple = (instType, instFree, instMax, instNumCpu, instRam, instDiskSpace)
397 instList.append(instTuple)
398 cluster['instances'] = instList
399 clusterList.append(cluster)
403 def get_rspec(api, creds, options):
405 # get slice's hrn from options
406 xrn = options.get('geni_slice_urn', '')
407 hrn, type = urn_to_hrn(xrn)
409 # get hrn of the original caller
410 origin_hrn = options.get('origin_hrn', None)
412 origin_hrn = Credential(string=creds[0]).get_gid_caller().get_hrn()
414 conn = getEucaConnection()
417 print >>sys.stderr, 'Error: Cannot create a connection to Eucalyptus'
418 return 'Cannot create a connection to Eucalyptus'
422 zones = conn.get_all_zones(['verbose'])
423 p = ZoneResultParser(zones)
425 cloud['clusters'] = clusters
428 images = conn.get_all_images()
429 cloud['images'] = images
432 keyPairs = conn.get_all_key_pairs()
433 cloud['keypairs'] = keyPairs
439 # Get the instances that belong to the given slice from sqlite3
440 # XXX use getOne() in production because the slice's hrn is supposed
441 # to be unique. For testing, uniqueness is turned off in the db.
442 # If the slice isn't found in the database, create a record for the
444 matchedSlices = list(Slice.select(Slice.q.slice_hrn == hrn))
446 theSlice = matchedSlices[-1]
448 theSlice = Slice(slice_hrn = hrn)
449 for instance in theSlice.instances:
450 instanceId.append(instance.instance_id)
452 # Get the information about those instances using their ids.
453 if len(instanceId) > 0:
454 reservations = conn.get_all_instances(instanceId)
457 for reservation in reservations:
458 for instance in reservation.instances:
459 instances.append(instance)
461 # Construct a dictionary for the EucaRSpecBuilder
463 for instance in instances:
464 instList = instancesDict.setdefault(instance.instance_type, [])
467 instInfoDict['id'] = instance.id
468 instInfoDict['public_dns'] = instance.public_dns_name
469 instInfoDict['state'] = instance.state
470 instInfoDict['key'] = instance.key_name
472 instList.append(instInfoDict)
473 cloud['instances'] = instancesDict
475 except EC2ResponseError, ec2RespErr:
476 errTree = ET.fromstring(ec2RespErr.body)
477 errMsgE = errTree.find('.//Message')
478 print >>sys.stderr, errMsgE.text
480 rspec = EucaRSpecBuilder(cloud).toXML()
482 # Remove the instances records so next time they won't
484 if 'instances' in cloud:
485 del cloud['instances']
490 Hook called via 'sfi.py create'
492 def create_slice(api, xrn, creds, xml, users):
494 hrn = urn_to_hrn(xrn)[0]
496 conn = getEucaConnection()
498 print >>sys.stderr, 'Error: Cannot create a connection to Eucalyptus'
502 schemaXML = ET.parse(EUCALYPTUS_RSPEC_SCHEMA)
503 rspecValidator = ET.RelaxNG(schemaXML)
504 rspecXML = ET.XML(xml)
505 if not rspecValidator(rspecXML):
506 error = rspecValidator.error_log.last_error
507 message = '%s (line %s)' % (error.message, error.line)
508 # XXX: InvalidRSpec is new. Currently, I am not working with Trunk code.
509 #raise InvalidRSpec(message)
510 raise Exception(message)
512 # Get the slice from db or create one.
513 s = Slice.select(Slice.q.slice_hrn == hrn).getOne(None)
515 s = Slice(slice_hrn = hrn)
517 # Process any changes in existing instance allocation
519 for sliceInst in s.instances:
520 pendingRmInst.append(sliceInst.instance_id)
521 existingInstGroup = rspecXML.findall('.//euca_instances')
522 for instGroup in existingInstGroup:
523 for existingInst in instGroup:
524 if existingInst.get('id') in pendingRmInst:
525 pendingRmInst.remove(existingInst.get('id'))
526 for inst in pendingRmInst:
527 print >>sys.stderr, 'Instance %s will be terminated' % inst
528 dbInst = EucaInstance.select(EucaInstance.q.instance_id == inst).getOne(None)
530 conn.terminate_instances(pendingRmInst)
532 # Process new instance requests
533 requests = rspecXML.findall('.//request')
535 # Get all the public keys associate with slice.
536 pubKeys = getKeysForSlice(s.slice_hrn)
537 print sys.stderr, "Passing the following keys to the instance:\n%s" % pubKeys
539 vmTypeElement = req.getparent()
540 instType = vmTypeElement.get('name')
541 numInst = int(req.find('instances').text)
542 instKernel = req.find('kernel_image').get('id')
543 instDiskImg = req.find('disk_image').get('id')
544 instKey = req.find('keypair').text
546 ramDiskElement = req.find('ramdisk')
547 ramDiskAttr = ramDiskElement.attrib
548 if 'id' in ramDiskAttr:
549 instRamDisk = ramDiskAttr['id']
553 # Create the instances
554 for i in range(0, numInst):
555 eucaInst = EucaInstance(slice = s,
556 kernel_id = instKernel,
557 image_id = instDiskImg,
558 ramdisk_id = instRamDisk,
560 inst_type = instType)
561 eucaInst.reserveInstance(conn, pubKeys)
569 #with open(sys.argv[1]) as xml:
570 # theRSpec = xml.read()
571 #create_slice(None, 'planetcloud.pc.test', theRSpec)
573 #rspec = get_rspec('euca', 'planetcloud.pc.test', 'planetcloud.pc.marcoy')
575 print getKeysForSlice('gc.gc.test1')
577 if __name__ == "__main__":