1 from __future__ import with_statement
7 from boto.ec2.regioninfo import RegionInfo
8 from boto.exception import EC2ResponseError
9 from ConfigParser import ConfigParser
10 from xmlbuilder import XMLBuilder
11 from lxml import etree as ET
12 from sqlobject import *
14 from sfa.util.faults import *
15 from sfa.util.xrn import urn_to_hrn, Xrn
16 from sfa.util.rspec import RSpec
17 from sfa.server.registry import Registries
18 from sfa.trust.credential import Credential
19 from sfa.plc.api import SfaAPI
20 from sfa.util.plxrn import hrn_to_pl_slicename, slicename_to_hrn
21 from sfa.util.callids import Callids
22 from sfa.util.sfalogging import sfa_logger
23 from sfa.rspecs.sfa_rspec import sfa_rspec_version
24 from sfa.util.version import version_core
27 # The data structure used to represent a cloud.
28 # It contains the cloud name, its ip address, image information,
29 # key pairs, and clusters information.
34 # The location of the RelaxNG schema.
36 EUCALYPTUS_RSPEC_SCHEMA='/etc/sfa/eucalyptus.rng'
39 sys.stderr = file('/var/log/euca_agg.log', 'a+')
43 # A representation of an Eucalyptus instance. This is a support class
44 # for instance <-> slice mapping.
46 class EucaInstance(SQLObject):
47 instance_id = StringCol(unique=True, default=None)
48 kernel_id = StringCol()
49 image_id = StringCol()
50 ramdisk_id = StringCol()
51 inst_type = StringCol()
52 key_pair = StringCol()
53 slice = ForeignKey('Slice')
56 # Contacts Eucalyptus and tries to reserve this instance.
58 # @param botoConn A connection to Eucalyptus.
59 # @param pubKeys A list of public keys for the instance.
61 def reserveInstance(self, botoConn, pubKeys):
62 print >>sys.stderr, 'Reserving an instance: image: %s, kernel: ' \
63 '%s, ramdisk: %s, type: %s, key: %s' % \
64 (self.image_id, self.kernel_id, self.ramdisk_id,
65 self.inst_type, self.key_pair)
67 # XXX The return statement is for testing. REMOVE in production
71 reservation = botoConn.run_instances(self.image_id,
72 kernel_id = self.kernel_id,
73 ramdisk_id = self.ramdisk_id,
74 instance_type = self.inst_type,
75 key_name = self.key_pair,
77 for instance in reservation.instances:
78 self.instance_id = instance.id
80 # If there is an error, destroy itself.
81 except EC2ResponseError, ec2RespErr:
82 errTree = ET.fromstring(ec2RespErr.body)
83 msg = errTree.find('.//Message')
84 print >>sys.stderr, msg.text
88 # A representation of a PlanetLab slice. This is a support class
89 # for instance <-> slice mapping.
91 class Slice(SQLObject):
92 slice_hrn = StringCol()
93 #slice_index = DatabaseIndex('slice_hrn')
94 instances = MultipleJoin('EucaInstance')
97 # Initialize the aggregate manager by reading a configuration file.
100 configParser = ConfigParser()
101 configParser.read(['/etc/sfa/eucalyptus_aggregate.conf', 'eucalyptus_aggregate.conf'])
102 if len(configParser.sections()) < 1:
103 print >>sys.stderr, 'No cloud defined in the config file'
104 raise Exception('Cannot find cloud definition in configuration file.')
106 # Only read the first section.
107 cloudSec = configParser.sections()[0]
108 cloud['name'] = cloudSec
109 cloud['access_key'] = configParser.get(cloudSec, 'access_key')
110 cloud['secret_key'] = configParser.get(cloudSec, 'secret_key')
111 cloud['cloud_url'] = configParser.get(cloudSec, 'cloud_url')
112 cloudURL = cloud['cloud_url']
113 if cloudURL.find('https://') >= 0:
114 cloudURL = cloudURL.replace('https://', '')
115 elif cloudURL.find('http://') >= 0:
116 cloudURL = cloudURL.replace('http://', '')
117 (cloud['ip'], parts) = cloudURL.split(':')
119 # Create image bundles
120 images = getEucaConnection().get_all_images()
121 cloud['images'] = images
122 cloud['imageBundles'] = {}
124 if i.type != 'machine' or i.kernel_id is None: continue
125 name = os.path.dirname(i.location)
126 detail = {'imageID' : i.id, 'kernelID' : i.kernel_id, 'ramdiskID' : i.ramdisk_id}
127 cloud['imageBundles'][name] = detail
129 # Initialize sqlite3 database.
130 dbPath = '/etc/sfa/db'
131 dbName = 'euca_aggregate.db'
133 if not os.path.isdir(dbPath):
134 print >>sys.stderr, '%s not found. Creating directory ...' % dbPath
137 conn = connectionForURI('sqlite://%s/%s' % (dbPath, dbName))
138 sqlhub.processConnection = conn
139 Slice.createTable(ifNotExists=True)
140 EucaInstance.createTable(ifNotExists=True)
142 # Make sure the schema exists.
143 if not os.path.exists(EUCALYPTUS_RSPEC_SCHEMA):
144 err = 'Cannot location schema at %s' % EUCALYPTUS_RSPEC_SCHEMA
145 print >>sys.stderr, err
149 # Creates a connection to Eucalytpus. This function is inspired by
150 # the make_connection() in Euca2ools.
152 # @return A connection object or None
154 def getEucaConnection():
156 accessKey = cloud['access_key']
157 secretKey = cloud['secret_key']
158 eucaURL = cloud['cloud_url']
163 if not accessKey or not secretKey or not eucaURL:
164 print >>sys.stderr, 'Please set ALL of the required environment ' \
165 'variables by sourcing the eucarc file.'
168 # Split the url into parts
169 if eucaURL.find('https://') >= 0:
171 eucaURL = eucaURL.replace('https://', '')
172 elif eucaURL.find('http://') >= 0:
174 eucaURL = eucaURL.replace('http://', '')
175 (eucaHost, parts) = eucaURL.split(':')
177 parts = parts.split('/')
178 eucaPort = int(parts[0])
180 srvPath = '/'.join(parts)
182 return boto.connect_ec2(aws_access_key_id=accessKey,
183 aws_secret_access_key=secretKey,
185 region=RegionInfo(None, 'eucalyptus', eucaHost),
190 # Returns a string of keys that belong to the users of the given slice.
191 # @param sliceHRN The hunman readable name of the slice.
194 def getKeysForSlice(api, sliceHRN):
195 cred = api.getCredential()
196 registry = api.registries[api.hrn]
199 # Get the slice record
200 records = registry.Resolve(sliceHRN, cred)
202 print >>sys.stderr, 'Cannot find any record for slice %s' % sliceHRN
205 # Find who can log into this slice
206 persons = records[0]['persons']
208 # Extract the keys from persons records
210 sliceUser = registry.Resolve(p, cred)
211 userKeys = sliceUser[0]['keys']
217 # A class that builds the RSpec for Eucalyptus.
219 class EucaRSpecBuilder(object):
221 # Initizes a RSpec builder
223 # @param cloud A dictionary containing data about a
224 # cloud (ex. clusters, ip)
225 def __init__(self, cloud):
226 self.eucaRSpec = XMLBuilder(format = True, tab_step = " ")
227 self.cloudInfo = cloud
230 # Creates a request stanza.
232 # @param num The number of instances to create.
233 # @param image The disk image id.
234 # @param kernel The kernel image id.
235 # @param keypair Key pair to embed.
236 # @param ramdisk Ramdisk id (optional).
238 def __requestXML(self, num, image, kernel, keypair, ramdisk = ''):
243 with xml.kernel_image(id=kernel):
249 with xml.ramdisk(id=ramdisk):
251 with xml.disk_image(id=image):
257 # Creates the cluster stanza.
259 # @param clusters Clusters information.
261 def __clustersXML(self, clusters):
262 cloud = self.cloudInfo
265 for cluster in clusters:
266 instances = cluster['instances']
267 with xml.cluster(id=cluster['name']):
271 for inst in instances:
272 with xml.vm_type(name=inst[0]):
275 with xml.max_instances:
279 with xml.memory(unit='MB'):
281 with xml.disk_space(unit='GB'):
283 if 'instances' in cloud and inst[0] in cloud['instances']:
284 existingEucaInstances = cloud['instances'][inst[0]]
285 with xml.euca_instances:
286 for eucaInst in existingEucaInstances:
287 with xml.euca_instance(id=eucaInst['id']):
289 xml << eucaInst['state']
291 xml << eucaInst['public_dns']
293 def __imageBundleXML(self, bundles):
296 for bundle in bundles.keys():
297 with xml.bundle(id=bundle):
301 # Creates the Images stanza.
303 # @param images A list of images in Eucalyptus.
305 def __imagesXML(self, images):
309 with xml.image(id=image.id):
313 xml << image.architecture
317 xml << image.location
320 # Creates the KeyPairs stanza.
322 # @param keypairs A list of key pairs in Eucalyptus.
324 def __keyPairsXML(self, keypairs):
332 # Generates the RSpec.
335 if not self.cloudInfo:
336 print >>sys.stderr, 'No cloud information'
340 cloud = self.cloudInfo
341 with xml.RSpec(type='eucalyptus'):
342 with xml.network(id=cloud['name']):
345 #self.__keyPairsXML(cloud['keypairs'])
346 #self.__imagesXML(cloud['images'])
347 self.__imageBundleXML(cloud['imageBundles'])
348 self.__clustersXML(cloud['clusters'])
352 # A parser to parse the output of availability-zones.
354 # Note: Only one cluster is supported. If more than one, this will
357 class ZoneResultParser(object):
358 def __init__(self, zones):
362 if len(self.zones) < 3:
368 cluster['name'] = self.zones[0].name
369 cluster['ip'] = self.zones[0].state
371 for i in range(2, len(self.zones)):
372 currZone = self.zones[i]
373 instType = currZone.name.split()[1]
375 stateString = currZone.state.split('/')
376 rscString = stateString[1].split()
378 instFree = int(stateString[0])
379 instMax = int(rscString[0])
380 instNumCpu = int(rscString[1])
381 instRam = int(rscString[2])
382 instDiskSpace = int(rscString[3])
384 instTuple = (instType, instFree, instMax, instNumCpu, instRam, instDiskSpace)
385 instList.append(instTuple)
386 cluster['instances'] = instList
387 clusterList.append(cluster)
391 def ListResources(api, creds, options, call_id):
392 if Callids().already_handled(call_id): return ""
394 # get slice's hrn from options
395 xrn = options.get('geni_slice_urn', '')
396 hrn, type = urn_to_hrn(xrn)
398 # get hrn of the original caller
399 origin_hrn = options.get('origin_hrn', None)
401 origin_hrn = Credential(string=creds).get_gid_caller().get_hrn()
402 # origin_hrn = Credential(string=creds[0]).get_gid_caller().get_hrn()
404 conn = getEucaConnection()
407 print >>sys.stderr, 'Error: Cannot create a connection to Eucalyptus'
408 return 'Cannot create a connection to Eucalyptus'
412 zones = conn.get_all_zones(['verbose'])
413 p = ZoneResultParser(zones)
415 cloud['clusters'] = clusters
418 images = conn.get_all_images()
419 cloud['images'] = images
420 cloud['imageBundles'] = {}
422 if i.type != 'machine' or i.kernel_id is None: continue
423 name = os.path.dirname(i.location)
424 detail = {'imageID' : i.id, 'kernelID' : i.kernel_id, 'ramdiskID' : i.ramdisk_id}
425 cloud['imageBundles'][name] = detail
428 keyPairs = conn.get_all_key_pairs()
429 cloud['keypairs'] = keyPairs
435 # Get the instances that belong to the given slice from sqlite3
436 # XXX use getOne() in production because the slice's hrn is supposed
437 # to be unique. For testing, uniqueness is turned off in the db.
438 # If the slice isn't found in the database, create a record for the
440 matchedSlices = list(Slice.select(Slice.q.slice_hrn == hrn))
442 theSlice = matchedSlices[-1]
444 theSlice = Slice(slice_hrn = hrn)
445 for instance in theSlice.instances:
446 instanceId.append(instance.instance_id)
448 # Get the information about those instances using their ids.
449 if len(instanceId) > 0:
450 reservations = conn.get_all_instances(instanceId)
453 for reservation in reservations:
454 for instance in reservation.instances:
455 instances.append(instance)
457 # Construct a dictionary for the EucaRSpecBuilder
459 for instance in instances:
460 instList = instancesDict.setdefault(instance.instance_type, [])
463 instInfoDict['id'] = instance.id
464 instInfoDict['public_dns'] = instance.public_dns_name
465 instInfoDict['state'] = instance.state
466 instInfoDict['key'] = instance.key_name
468 instList.append(instInfoDict)
469 cloud['instances'] = instancesDict
471 except EC2ResponseError, ec2RespErr:
472 errTree = ET.fromstring(ec2RespErr.body)
473 errMsgE = errTree.find('.//Message')
474 print >>sys.stderr, errMsgE.text
476 rspec = EucaRSpecBuilder(cloud).toXML()
478 # Remove the instances records so next time they won't
480 if 'instances' in cloud:
481 del cloud['instances']
486 Hook called via 'sfi.py create'
488 def CreateSliver(api, xrn, creds, xml, users, call_id):
489 if Callids().already_handled(call_id): return ""
492 hrn = urn_to_hrn(xrn)[0]
494 conn = getEucaConnection()
496 print >>sys.stderr, 'Error: Cannot create a connection to Eucalyptus'
500 schemaXML = ET.parse(EUCALYPTUS_RSPEC_SCHEMA)
501 rspecValidator = ET.RelaxNG(schemaXML)
502 rspecXML = ET.XML(xml)
503 for network in rspecXML.iterfind("./network"):
504 if network.get('id') != cloud['name']:
505 # Throw away everything except my own RSpec
506 # sfa_logger().error("CreateSliver: deleting %s from rspec"%network.get('id'))
507 network.getparent().remove(network)
508 if not rspecValidator(rspecXML):
509 error = rspecValidator.error_log.last_error
510 message = '%s (line %s)' % (error.message, error.line)
511 # XXX: InvalidRSpec is new. Currently, I am not working with Trunk code.
512 #raise InvalidRSpec(message)
513 raise Exception(message)
515 # Get the slice from db or create one.
516 s = Slice.select(Slice.q.slice_hrn == hrn).getOne(None)
518 s = Slice(slice_hrn = hrn)
520 # Process any changes in existing instance allocation
522 for sliceInst in s.instances:
523 pendingRmInst.append(sliceInst.instance_id)
524 existingInstGroup = rspecXML.findall(".//euca_instances")
525 for instGroup in existingInstGroup:
526 for existingInst in instGroup:
527 if existingInst.get('id') in pendingRmInst:
528 pendingRmInst.remove(existingInst.get('id'))
529 for inst in pendingRmInst:
530 print >>sys.stderr, 'Instance %s will be terminated' % inst
531 dbInst = EucaInstance.select(EucaInstance.q.instance_id == inst).getOne(None)
533 conn.terminate_instances(pendingRmInst)
535 # Process new instance requests
536 requests = rspecXML.findall(".//request")
538 # Get all the public keys associate with slice.
539 pubKeys = getKeysForSlice(api, s.slice_hrn)
540 print >>sys.stderr, "Passing the following keys to the instance:\n%s" % pubKeys
543 vmTypeElement = req.getparent()
544 instType = vmTypeElement.get('name')
545 numInst = int(req.find('instances').text)
547 bundleName = req.find('bundle').text
548 if not cloud['imageBundles'][bundleName]:
549 print >>sys.stderr, 'Cannot find bundle %s' % bundleName
550 bundleInfo = cloud['imageBundles'][bundleName]
551 instKernel = bundleInfo['kernelID']
552 instDiskImg = bundleInfo['imageID']
553 instRamDisk = bundleInfo['ramdiskID']
556 # Create the instances
557 for i in range(0, numInst):
558 eucaInst = EucaInstance(slice = s,
559 kernel_id = instKernel,
560 image_id = instDiskImg,
561 ramdisk_id = instRamDisk,
563 inst_type = instType)
564 eucaInst.reserveInstance(conn, pubKeys)
566 # xxx - should return altered rspec
567 # with enough data for the client to understand what's happened
572 request_rspec_versions = [dict(sfa_rspec_version)]
573 ad_rspec_versions = [dict(sfa_rspec_version)]
574 version_more = {'interface':'aggregate',
577 'request_rspec_versions': request_rspec_versions,
578 'ad_rspec_versions': ad_rspec_versions,
579 'default_ad_rspec': dict(sfa_rspec_version)
581 return version_core(version_more)
587 #with open(sys.argv[1]) as xml:
588 # theRSpec = xml.read()
589 #CreateSliver(None, 'planetcloud.pc.test', theRSpec, 'call-id-cloudtest')
591 #rspec = ListResources('euca', 'planetcloud.pc.test', 'planetcloud.pc.marcoy', 'test_euca')
594 server_key_file = '/var/lib/sfa/authorities/server.key'
595 server_cert_file = '/var/lib/sfa/authorities/server.cert'
596 api = SfaAPI(key_file = server_key_file, cert_file = server_cert_file, interface='aggregate')
597 print getKeysForSlice(api, 'gc.gc.test1')
599 if __name__ == "__main__":