1 from __future__ import with_statement
2 from sfa.util.faults import *
3 from sfa.util.namespace import *
4 from sfa.util.rspec import RSpec
5 from sfa.server.registry import Registries
8 from boto.ec2.regioninfo import RegionInfo
9 from boto.exception import EC2ResponseError
10 from ConfigParser import ConfigParser
11 from xmlbuilder import XMLBuilder
12 from lxml import etree as ET
13 from sqlobject import *
19 # The data structure used to represent a cloud.
20 # It contains the cloud name, its ip address, image information,
21 # key pairs, and clusters information.
26 # The location of the RelaxNG schema.
28 EUCALYPTUS_RSPEC_SCHEMA='/etc/sfa/eucalyptus.rng'
31 # A representation of an Eucalyptus instance. This is a support class
32 # for instance <-> slice mapping.
34 class EucaInstance(SQLObject):
35 instance_id = StringCol(unique=True, default=None)
36 kernel_id = StringCol()
37 image_id = StringCol()
38 ramdisk_id = StringCol()
39 inst_type = StringCol()
40 key_pair = StringCol()
41 slice = ForeignKey('Slice')
44 # Contacts Eucalyptus and tries to reserve this instance.
46 # @param botoConn A connection to Eucalyptus.
48 def reserveInstance(self, botoConn):
49 print >>sys.stderr, 'Reserving an instance: image: %s, kernel: ' \
50 '%s, ramdisk: %s, type: %s, key: %s' % \
51 (self.image_id, self.kernel_id, self.ramdisk_id,
52 self.inst_type, self.key_pair)
54 # XXX The return statement is for testing. REMOVE in production
58 reservation = botoConn.run_instances(self.image_id,
59 kernel_id = self.kernel_id,
60 ramdisk_id = self.ramdisk_id,
61 instance_type = self.inst_type,
62 key_name = self.key_pair)
63 for instance in reservation.instances:
64 self.instance_id = instance.id
66 # If there is an error, destroy itself.
67 except EC2ResponseError, ec2RespErr:
68 errTree = ET.fromstring(ec2RespErr.body)
69 msg = errTree.find('.//Message')
70 print >>sys.stderr, msg.text
74 # A representation of a PlanetLab slice. This is a support class
75 # for instance <-> slice mapping.
77 class Slice(SQLObject):
78 slice_hrn = StringCol()
79 #slice_index = DatabaseIndex('slice_hrn')
80 instances = MultipleJoin('EucaInstance')
83 # Initialize the aggregate manager by reading a configuration file.
86 configParser = ConfigParser()
87 configParser.read(['/etc/sfa/eucalyptus_aggregate.conf', 'eucalyptus_aggregate.conf'])
88 if len(configParser.sections()) < 1:
89 print >>sys.stderr, 'No cloud defined in the config file'
90 raise Exception('Cannot find cloud definition in configuration file.')
92 # Only read the first section.
93 cloudSec = configParser.sections()[0]
94 cloud['name'] = cloudSec
95 cloud['access_key'] = configParser.get(cloudSec, 'access_key')
96 cloud['secret_key'] = configParser.get(cloudSec, 'secret_key')
97 cloud['cloud_url'] = configParser.get(cloudSec, 'cloud_url')
98 cloudURL = cloud['cloud_url']
99 if cloudURL.find('https://') >= 0:
100 cloudURL = cloudURL.replace('https://', '')
101 elif cloudURL.find('http://') >= 0:
102 cloudURL = cloudURL.replace('http://', '')
103 (cloud['ip'], parts) = cloudURL.split(':')
105 # Initialize sqlite3 database.
106 dbPath = '/etc/sfa/db'
107 dbName = 'euca_aggregate.db'
109 if not os.path.isdir(dbPath):
110 print >>sys.stderr, '%s not found. Creating directory ...' % dbPath
113 conn = connectionForURI('sqlite://%s/%s' % (dbPath, dbName))
114 sqlhub.processConnection = conn
115 Slice.createTable(ifNotExists=True)
116 EucaInstance.createTable(ifNotExists=True)
118 # Make sure the schema exists.
119 if not os.path.exists(EUCALYPTUS_RSPEC_SCHEMA):
120 err = 'Cannot location schema at %s' % EUCALYPTUS_RSPEC_SCHEMA
121 print >>sys.stderr, err
125 # Creates a connection to Eucalytpus. This function is inspired by
126 # the make_connection() in Euca2ools.
128 # @return A connection object or None
130 def getEucaConnection():
132 accessKey = cloud['access_key']
133 secretKey = cloud['secret_key']
134 eucaURL = cloud['cloud_url']
139 if not accessKey or not secretKey or not eucaURL:
140 print >>sys.stderr, 'Please set ALL of the required environment ' \
141 'variables by sourcing the eucarc file.'
144 # Split the url into parts
145 if eucaURL.find('https://') >= 0:
147 eucaURL = eucaURL.replace('https://', '')
148 elif eucaURL.find('http://') >= 0:
150 eucaURL = eucaURL.replace('http://', '')
151 (eucaHost, parts) = eucaURL.split(':')
153 parts = parts.split('/')
154 eucaPort = int(parts[0])
156 srvPath = '/'.join(parts)
158 return boto.connect_ec2(aws_access_key_id=accessKey,
159 aws_secret_access_key=secretKey,
161 region=RegionInfo(None, 'eucalyptus', eucaHost),
166 # A class that builds the RSpec for Eucalyptus.
168 class EucaRSpecBuilder(object):
170 # Initizes a RSpec builder
172 # @param cloud A dictionary containing data about a
173 # cloud (ex. clusters, ip)
174 def __init__(self, cloud):
175 self.eucaRSpec = XMLBuilder(format = True, tab_step = " ")
176 self.cloudInfo = cloud
179 # Creates a request stanza.
181 # @param num The number of instances to create.
182 # @param image The disk image id.
183 # @param kernel The kernel image id.
184 # @param keypair Key pair to embed.
185 # @param ramdisk Ramdisk id (optional).
187 def __requestXML(self, num, image, kernel, keypair, ramdisk = ''):
192 with xml.kernel_image(id=kernel):
198 with xml.ramdisk(id=ramdisk):
200 with xml.disk_image(id=image):
206 # Creates the cluster stanza.
208 # @param clusters Clusters information.
210 def __clustersXML(self, clusters):
211 cloud = self.cloudInfo
214 for cluster in clusters:
215 instances = cluster['instances']
216 with xml.cluster(id=cluster['name']):
220 for inst in instances:
221 with xml.vm_type(name=inst[0]):
224 with xml.max_instances:
228 with xml.memory(unit='MB'):
230 with xml.disk_space(unit='GB'):
232 if inst[0] == 'm1.small':
233 self.__requestXML(1, 'emi-88760F45', 'eki-F26610C6', 'cortex')
234 if 'instances' in cloud and inst[0] in cloud['instances']:
235 existingEucaInstances = cloud['instances'][inst[0]]
236 with xml.euca_instances:
237 for eucaInst in existingEucaInstances:
238 with xml.euca_instance(id=eucaInst['id']):
240 xml << eucaInst['state']
242 xml << eucaInst['public_dns']
244 xml << eucaInst['key']
247 # Creates the Images stanza.
249 # @param images A list of images in Eucalyptus.
251 def __imagesXML(self, images):
255 with xml.image(id=image.id):
259 xml << image.architecture
263 xml << image.location
266 # Creates the KeyPairs stanza.
268 # @param keypairs A list of key pairs in Eucalyptus.
270 def __keyPairsXML(self, keypairs):
278 # Generates the RSpec.
281 if not self.cloudInfo:
282 print >>sys.stderr, 'No cloud information'
286 cloud = self.cloudInfo
287 with xml.RSpec(type='eucalyptus'):
288 with xml.cloud(id=cloud['name']):
291 self.__keyPairsXML(cloud['keypairs'])
292 self.__imagesXML(cloud['images'])
293 self.__clustersXML(cloud['clusters'])
297 # A parser to parse the output of availability-zones.
299 # Note: Only one cluster is supported. If more than one, this will
302 class ZoneResultParser(object):
303 def __init__(self, zones):
307 if len(self.zones) < 3:
313 cluster['name'] = self.zones[0].name
314 cluster['ip'] = self.zones[0].state
316 for i in range(2, len(self.zones)):
317 currZone = self.zones[i]
318 instType = currZone.name.split()[1]
320 stateString = currZone.state.split('/')
321 rscString = stateString[1].split()
323 instFree = int(stateString[0])
324 instMax = int(rscString[0])
325 instNumCpu = int(rscString[1])
326 instRam = int(rscString[2])
327 instDiskSpace = int(rscString[3])
329 instTuple = (instType, instFree, instMax, instNumCpu, instRam, instDiskSpace)
330 instList.append(instTuple)
331 cluster['instances'] = instList
332 clusterList.append(cluster)
336 def get_rspec(api, creds, options):
338 # get slice's hrn from options
339 xrn = options.get('geni_slice_urn', None)
340 hrn, type = urn_to_hrn(xrn)
342 # get hrn of the original caller
343 origin_hrn = options.get('origin_hrn', None)
345 origin_hrn = Credential(string=creds[0]).get_gid_caller().get_hrn()
347 conn = getEucaConnection()
350 print >>sys.stderr, 'Error: Cannot create a connection to Eucalyptus'
351 return 'Cannot create a connection to Eucalyptus'
355 zones = conn.get_all_zones(['verbose'])
356 p = ZoneResultParser(zones)
358 cloud['clusters'] = clusters
361 images = conn.get_all_images()
362 cloud['images'] = images
365 keyPairs = conn.get_all_key_pairs()
366 cloud['keypairs'] = keyPairs
372 # Get the instances that belong to the given slice from sqlite3
373 # XXX use getOne() in production because the slice's hrn is supposed
374 # to be unique. For testing, uniqueness is turned off in the db.
375 # If the slice isn't found in the database, create a record for the
377 matchedSlices = list(Slice.select(Slice.q.slice_hrn == hrn))
379 theSlice = matchedSlices[-1]
381 theSlice = Slice(slice_hrn = hrn)
382 for instance in theSlice.instances:
383 instanceId.append(instance.instance_id)
385 # Get the information about those instances using their ids.
386 if len(instanceId) > 0:
387 reservations = conn.get_all_instances(instanceId)
390 for reservation in reservations:
391 for instance in reservation.instances:
392 instances.append(instance)
394 # Construct a dictory for the EucaRSpecBuilder
396 for instance in instances:
397 instList = instancesDict.setdefault(instance.instance_type, [])
400 instInfoDict['id'] = instance.id
401 instInfoDict['public_dns'] = instance.public_dns_name
402 instInfoDict['state'] = instance.state
403 instInfoDict['key'] = instance.key_name
405 instList.append(instInfoDict)
406 cloud['instances'] = instancesDict
408 except EC2ResponseError, ec2RespErr:
409 errTree = ET.fromstring(ec2RespErr.body)
410 errMsgE = errTree.find('.//Message')
411 print >>sys.stderr, errMsgE.text
413 rspec = EucaRSpecBuilder(cloud).toXML()
415 # Remove the instances records so next time they won't
417 if 'instances' in cloud:
418 del cloud['instances']
423 Hook called via 'sfi.py create'
425 def create_slice(api, xrn, creds, xml, users):
427 hrn = urn_to_hrn(xrn)[0]
429 conn = getEucaConnection()
431 print >>sys.stderr, 'Error: Cannot create a connection to Eucalyptus'
435 schemaXML = ET.parse(EUCALYPTUS_RSPEC_SCHEMA)
436 rspecValidator = ET.RelaxNG(schemaXML)
437 rspecXML = ET.XML(xml)
438 if not rspecValidator(rspecXML):
439 error = rspecValidator.error_log.last_error
440 message = '%s (line %s)' % (error.message, error.line)
441 # XXX: InvalidRSpec is new. Currently, I am not working with Trunk code.
442 #raise InvalidRSpec(message)
443 raise Exception(message)
445 # Get the slice from db or create one.
446 s = Slice.select(Slice.q.slice_hrn == hrn).getOne(None)
448 s = Slice(slice_hrn = hrn)
450 # Process any changes in existing instance allocation
452 for sliceInst in s.instances:
453 pendingRmInst.append(sliceInst.instance_id)
454 existingInstGroup = rspecXML.findall('.//euca_instances')
455 for instGroup in existingInstGroup:
456 for existingInst in instGroup:
457 if existingInst.get('id') in pendingRmInst:
458 pendingRmInst.remove(existingInst.get('id'))
459 for inst in pendingRmInst:
460 print >>sys.stderr, 'Instance %s will be terminated' % inst
461 dbInst = EucaInstance.select(EucaInstance.q.instance_id == inst).getOne(None)
463 conn.terminate_instances(pendingRmInst)
465 # Process new instance requests
466 requests = rspecXML.findall('.//request')
468 vmTypeElement = req.getparent()
469 instType = vmTypeElement.get('name')
470 numInst = int(req.find('instances').text)
471 instKernel = req.find('kernel_image').get('id')
472 instDiskImg = req.find('disk_image').get('id')
473 instKey = req.find('keypair').text
475 ramDiskElement = req.find('ramdisk')
476 ramDiskAttr = ramDiskElement.attrib
477 if 'id' in ramDiskAttr:
478 instRamDisk = ramDiskAttr['id']
482 # Create the instances
483 for i in range(0, numInst):
484 eucaInst = EucaInstance(slice = s,
485 kernel_id = instKernel,
486 image_id = instDiskImg,
487 ramdisk_id = instRamDisk,
489 inst_type = instType)
490 eucaInst.reserveInstance(conn)
498 with open(sys.argv[1]) as xml:
499 theRSpec = xml.read()
500 create_slice(None, 'planetcloud.pc.test', theRSpec)
502 #rspec = get_rspec('euca', 'planetcloud.pc.test', 'planetcloud.pc.marcoy')
505 if __name__ == "__main__":