Merge Master in geni-v3 conflict resolution sfa-3.0-0
authorMohamed Larabi <mohamed.larabi@inria.fr>
Mon, 25 Mar 2013 16:06:12 +0000 (17:06 +0100)
committerMohamed Larabi <mohamed.larabi@inria.fr>
Mon, 25 Mar 2013 16:06:12 +0000 (17:06 +0100)
19 files changed:
1  2 
init.d/sfa
setup.py
sfa/client/client_helper.py
sfa/client/sfaadmin.py
sfa/client/sfi.py
sfa/importer/plimporter.py
sfa/managers/registry_manager.py
sfa/openstack/osaggregate.py
sfa/planetlab/plaggregate.py
sfa/planetlab/pldriver.py
sfa/planetlab/plslices.py
sfa/rspecs/elements/node.py
sfa/rspecs/elements/versions/pgv2Node.py
sfa/rspecs/elements/versions/sfav1Lease.py
sfa/rspecs/elements/versions/sfav1Node.py
sfa/rspecs/rspec.py
sfa/server/sfaapi.py
sfa/storage/model.py
sfa/trust/credential.py

diff --cc init.d/sfa
Simple merge
diff --cc setup.py
Simple merge
Simple merge
Simple merge
@@@ -281,17 -250,17 +259,17 @@@ class Sfi
          ("version", ""),  
          ("list", "authority"),
          ("show", "name"),
-         ("add", "record"),
-         ("update", "record"),
+         ("add", "[record]"),
+         ("update", "[record]"),
          ("remove", "name"),
 -        ("slices", ""),
 -        ("resources", "[slice_hrn]"),
 +        ("resources", ""),
 +        ("describe", "slice_hrn"),
          ("create", "slice_hrn rspec"),
 +        ("allocate", "slice_hrn rspec"),
 +        ("provision", "slice_hrn"),
 +        ("action", "slice_hrn action"), 
          ("delete", "slice_hrn"),
          ("status", "slice_hrn"),
 -        ("start", "slice_hrn"),
 -        ("stop", "slice_hrn"),
 -        ("reset", "slice_hrn"),
          ("renew", "slice_hrn time"),
          ("shutdown", "slice_hrn"),
          ("get_ticket", "slice_hrn rspec"),
@@@ -992,12 -941,9 +987,9 @@@ or version information about sfi itsel
          server = self.sliceapi()
          # creds
          creds = [self.my_credential_string]
-         if options.delegate:
-             delegated_cred = self.delegate_cred(self.my_credential_string, get_authority(self.authority))
-             creds.append(delegated_cred)  
          # options and call_id when supported
          api_options = {}
 -      api_options['call_id']=unique_call_id()
 +        api_options['call_id']=unique_call_id()
          if options.show_credential:
              show_credentials(creds)
          result = server.ListSlices(creds, *self.ois(server,api_options))
@@@ -1181,11 -1101,8 +1177,8 @@@ or with an slice hrn, shows currently p
          slice_urn = hrn_to_urn(slice_hrn, 'slice') 
  
          # creds
 -        slice_cred = self.slice_credential_string(slice_hrn)
 +        slice_cred = self.slice_credential(slice_hrn)
          creds = [slice_cred]
-         if options.delegate:
-             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
-             creds.append(delegated_cred)
          
          # options and call_id when supported
          api_options = {}
          slice_urn = hrn_to_urn(slice_hrn, 'slice') 
  
          # creds 
 -        slice_cred = self.slice_credential_string(slice_hrn)
 +        slice_cred = self.slice_credential(slice_hrn)
          creds = [slice_cred]
-         if options.delegate:
-             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
-             creds.append(delegated_cred)
  
          # options and call_id when supported
          api_options = {}
          slice_urn = hrn_to_urn(slice_hrn, 'slice') 
          # time: don't try to be smart on the time format, server-side will
          # creds
 -        slice_cred = self.slice_credential_string(args[0])
 +        slice_cred = self.slice_credential(args[0])
          creds = [slice_cred]
-         if options.delegate:
-             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
-             creds.append(delegated_cred)
          # options and call_id when supported
          api_options = {}
 -      api_options['call_id']=unique_call_id()
 +        api_options['call_id']=unique_call_id()
          if options.show_credential:
              show_credentials(creds)
 -        result =  server.RenewSliver(slice_urn, creds, input_time, *self.ois(server,api_options))
 +        result =  server.Renew([slice_urn], creds, input_time, *self.ois(server,api_options))
          value = ReturnValue.get_value(result)
          if self.options.raw:
              save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
          slice_hrn = args[0]
          slice_urn = hrn_to_urn(slice_hrn, 'slice') 
          # creds
 -        slice_cred = self.slice_credential_string(slice_hrn)
 +        slice_cred = self.slice_credential(slice_hrn)
          creds = [slice_cred]
-         if options.delegate:
-             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
-             creds.append(delegated_cred)
          result = server.Shutdown(slice_urn, creds)
          value = ReturnValue.get_value(result)
          if self.options.raw:
@@@ -250,13 -254,20 +254,20 @@@ class PlImporter
                      pass
                  node_record.stale=False
  
 -            site_pis=[]
 +            site_pis=set()
              # import persons
              for person_id in site['person_ids']:
-                 try:
-                     person = persons_by_id[person_id]
-                 except:
-                     self.logger.warning ("PlImporter: cannot locate person_id %s - ignored"%person_id)
+                 proceed=False
+                 if person_id in persons_by_id:
+                     person=persons_by_id[person_id]
+                     proceed=True
+                 elif person_id in disabled_person_ids:
+                     pass
+                 else:
+                     self.logger.warning ("PlImporter: cannot locate person_id %s in site %s - ignored"%(person_id,site_hrn))
+                 # make sure to NOT run this if anything is wrong
+                 if not proceed: continue
                  person_hrn = email_to_hrn(site_hrn, person['email'])
                  # xxx suspicious again
                  if len(person_hrn) > 64: person_hrn = person_hrn[:64]
                      self.logger.log_exc("PlImporter: failed to import person %d %s"%(person['person_id'],person['email']))
      
              # maintain the list of PIs for a given site
 -            # for the record, Jordan had proposed the following addition as a welcome hotfix to a previous version:
 -            # site_pis = list(set(site_pis)) 
 -            # this was likely due to a bug in the above logic, that had to do with disabled persons
 -            # being improperly handled, and where the whole loop on persons
 -            # could be performed twice with the same person...
 -            # so hopefully we do not need to eliminate duplicates explicitly here anymore
 +            site_record.reg_pis = list(site_pis)
+             site_record.reg_pis = site_pis
              dbsession.commit()
  
              # import slices
                          self.logger.info("PlImporter: imported slice: %s" % slice_record)  
                          self.remember_record ( slice_record )
                      except:
-                         self.logger.log_exc("PlImporter: failed to import slice")
+                         self.logger.log_exc("PlImporter: failed to import slice %s (%s)"%(slice_hrn,slice['name']))
                  else:
 +                    # update the pointer if it has changed
 +                    if slice_id != slice_record.pointer:
 +                        self.logger.info("updating record (slice) pointer")
 +                        slice_record.pointer = slice_id
 +                        dbsession.commit()             
                      # xxx update the record ...
 -                    # given that we record the current set of users anyways, there does not seem to be much left to do here
 -                    # self.logger.warning ("Slice update not yet implemented on slice %s (%s)"%(slice_hrn,slice['name']))
 -                    pass
 +                    #self.logger.warning ("Slice update not yet implemented")
                  # record current users affiliated with the slice
                  slice_record.reg_researchers = \
                      [ self.locate_by_type_pointer ('user',user_id) for user_id in slice['person_ids'] ]
Simple merge
@@@ -194,57 -157,21 +195,56 @@@ class OSAggregate
                             'port':'22', 'username': 'root'})
              service = Services({'login': login})
              rspec_node['services'].append(service)
 -        return rspec_node 
 +        return rspec_node
  
 -    def get_slice_nodes(self, slice_xrn):
 -        # update nova connection
 -        tenant_name = OSXrn(xrn=slice_xrn, type='slice').get_tenant_name()
 -        self.driver.shell.nova_manager.connect(tenant=tenant_name)    
 -        
 -        zones = self.get_availability_zones()
 -        name = hrn_to_os_slicename(slice_xrn)
 -        instances = self.driver.shell.nova_manager.servers.findall(name=name)
 -        rspec_nodes = []
 -        for instance in instances:
 -            rspec_nodes.append(self.instance_to_rspec_node(slice_xrn, instance))
 -        return rspec_nodes
  
 +    def instance_to_sliver(self, instance, xrn=None):
 +        if xrn:
 +            sliver_hrn = '%s.%s' % (self.driver.hrn, instance.id)
 +            sliver_id = Xrn(sliver_hrn, type='sliver').urn
 +
 +        sliver = Sliver({'sliver_id': sliver_id,
 +                         'name': instance.name,
 +                         'type': instance.name,
 +                         'cpus': str(instance.vcpus),
 +                         'memory': str(instance.ram),
 +                         'storage':  str(instance.disk)})
 +        return sliver   
 +
 +    def instance_to_geni_sliver(self, instance, sliver_allocations = {}):
 +        sliver_hrn = '%s.%s' % (self.driver.hrn, instance.id)
 +        sliver_id = Xrn(sliver_hrn, type='sliver').urn
 + 
 +        # set sliver allocation and operational status
 +        sliver_allocation = sliver_allocations[sliver_id]
 +        if sliver_allocation:
 +            allocation_status = sliver_allocation.allocation_state
 +            if allocation_status == 'geni_allocated':
 +                op_status =  'geni_pending_allocation'
 +            elif allocation_status == 'geni_provisioned':
 +                state = instance.state.lower()
 +                if state == 'active':
 +                    op_status = 'geni_ready'
 +                elif state == 'building':
 +                    op_status = 'geni_notready'
 +                elif state == 'failed':
 +                    op_status =' geni_failed'
 +                else:
 +                    op_status = 'geni_unknown'
 +            else:
 +                allocation_status = 'geni_unallocated'    
 +        # required fields
 +        geni_sliver = {'geni_sliver_urn': sliver_id, 
 +                       'geni_expires': None,
 +                       'geni_allocation_status': allocation_status,
 +                       'geni_operational_status': op_status,
 +                       'geni_error': None,
 +                       'plos_created_at': datetime_to_string(utcparse(instance.created)),
 +                       'plos_sliver_type': self.shell.nova_manager.flavors.find(id=instance.flavor['id']).name,
 +                        }
 +
 +        return geni_sliver
 +                        
      def get_aggregate_nodes(self):
          zones = self.get_availability_zones()
          # available sliver/instance/vm types
          files = {'/root/.ssh/authorized_keys': authorized_keys}
          rspec = RSpec(rspec)
          requested_instances = defaultdict(list)
+         
          # iterate over clouds/zones/nodes
 -        created_instances = []
 +        slivers = []
          for node in rspec.version.get_nodes_with_slivers():
              instances = node.get('slivers', [])
              if not instances:
@@@ -112,215 -99,170 +112,218 @@@ class PlAggregate
              pl_initscripts[initscript['initscript_id']] = initscript
          return pl_initscripts
  
 +    def get_slivers(self, urns, options={}):
 +        names = set()
 +        slice_ids = set()
 +        node_ids = []
 +        for urn in urns:
 +            xrn = PlXrn(xrn=urn)
 +            if xrn.type == 'sliver':
 +                 # id: slice_id-node_id
 +                try:
 +                    sliver_id_parts = xrn.get_sliver_id_parts()
 +                    slice_id = int(sliver_id_parts[0]) 
 +                    node_id = int(sliver_id_parts[1])
 +                    slice_ids.add(slice_id) 
 +                    node_ids.append(node_id)
 +                except ValueError:
 +                    pass 
 +            else:  
 +                names.add(xrn.pl_slicename())
  
 -    def get_slice_and_slivers(self, slice_xrn):
 -        """
 -        Returns a dict of slivers keyed on the sliver's node_id
 -        """
 -        slivers = {}
 -        slice = None
 -        if not slice_xrn:
 -            return (slice, slivers)
 -        slice_urn = hrn_to_urn(slice_xrn, 'slice')
 -        slice_hrn, _ = urn_to_hrn(slice_xrn)
 -        slice_name = hrn_to_pl_slicename(slice_hrn)
 -        slices = self.driver.shell.GetSlices(slice_name)
 +        filter = {}
 +        if names:
 +            filter['name'] = list(names)
 +        if slice_ids:
 +            filter['slice_id'] = list(slice_ids)
 +        # get slices
 +        slices = self.driver.shell.GetSlices(filter)
          if not slices:
 -            return (slice, slivers)
 -        slice = slices[0]
 -
 -        # sort slivers by node id    
 -        for node_id in slice['node_ids']:
 -            sliver_xrn = Xrn(slice_urn, type='sliver', id=node_id)
 -            sliver_xrn.set_authority(self.driver.hrn)
 -            sliver = Sliver({'sliver_id': sliver_xrn.urn,
 -                             'name': slice['name'],
 -                             'type': 'plab-vserver', 
 -                             'tags': []})
 -            slivers[node_id]= sliver
 -
 -        # sort sliver attributes by node id    
 -        tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice['slice_tag_ids']})
 -        for tag in tags:
 -            # most likely a default/global sliver attribute (node_id == None)
 -            if tag['node_id'] not in slivers:
 -                sliver_xrn = Xrn(slice_urn, type='sliver', id=tag['node_id'])
 -                sliver_xrn.set_authority(self.driver.hrn)
 -                sliver = Sliver({'sliver_id': sliver_xrn.urn,
 -                                 'name': slice['name'],
 -                                 'type': 'plab-vserver',
 -                                 'tags': []})
 -                slivers[tag['node_id']] = sliver
 -            slivers[tag['node_id']]['tags'].append(tag)
 +            return []
 +        slice = slices[0]     
 +        slice['hrn'] = PlXrn(auth=self.driver.hrn, slicename=slice['name']).hrn   
 +
 +        # get sliver users
 +        persons = []
 +        person_ids = []
 +        for slice in slices:
 +            person_ids.extend(slice['person_ids'])
 +        if person_ids:
 +            persons = self.driver.shell.GetPersons(person_ids)
 +                 
 +        # get user keys
 +        keys = {}
 +        key_ids = []
 +        for person in persons:
 +            key_ids.extend(person['key_ids'])
          
 -        return (slice, slivers)
 -
 -    def get_nodes_and_links(self, slice_xrn, slice=None,slivers=[], options={}):
 -        # if we are dealing with a slice that has no node just return 
 -        # and empty list    
 -        if slice_xrn:
 -            if not slice or not slice['node_ids']:
 -                return ([],[])
 +        if key_ids:
 +            key_list = self.driver.shell.GetKeys(key_ids)
 +            for key in key_list:
 +                keys[key['key_id']] = key  
 +
 +        # construct user key info
 +        users = []
 +        for person in persons:
 +            name = person['email'][0:person['email'].index('@')]
 +            user = {
 +                'login': slice['name'], 
 +                'user_urn': Xrn('%s.%s' % (self.driver.hrn, name), type='user').urn,
 +                'keys': [keys[k_id]['key'] for k_id in person['key_ids'] if k_id in keys]
 +            }
 +            users.append(user)
 +
 +        if node_ids:
 +            node_ids = [node_id for node_id in node_ids if node_id in slice['node_ids']]
 +            slice['node_ids'] = node_ids
 +        tags_dict = self.get_slice_tags(slice)
 +        nodes_dict = self.get_slice_nodes(slice, options)
 +        slivers = []
 +        for node in nodes_dict.values():
 +            node.update(slice) 
 +            node['tags'] = tags_dict[node['node_id']]
 +            sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
 +            node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
 +            node['urn'] = node['sliver_id'] 
 +            node['services_user'] = users
 +            slivers.append(node)
 +        return slivers
 +
 +    def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}):
 +        rspec_node = NodeElement()
 +        # xxx how to retrieve site['login_base']
 +        site=sites[node['site_id']]
 +        rspec_node['component_id'] = PlXrn(self.driver.hrn, hostname=node['hostname']).get_urn()
 +        rspec_node['component_name'] = node['hostname']
 +        rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
 +        rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
 +        # do not include boot state (<available> element) in the manifest rspec
 +        rspec_node['boot_state'] = node['boot_state']
 +        if node['boot_state'] == 'boot': 
 +            rspec_node['available'] = 'true'
 +        else:
 +            rspec_node['available'] = 'false'
 +        rspec_node['exclusive'] = 'false'
 +        rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
 +                                        HardwareType({'name': 'pc'})]
 +        # only doing this because protogeni rspec needs
 +        # to advertise available initscripts
 +        rspec_node['pl_initscripts'] = pl_initscripts.values()
 +        # add site/interface info to nodes.
 +        # assumes that sites, interfaces and tags have already been prepared.
 +        if site['longitude'] and site['latitude']:
 +            location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
 +            rspec_node['location'] = location
 +        # Granularity
 +        granularity = Granularity({'grain': grain})
 +        rspec_node['granularity'] = granularity
 +        rspec_node['interfaces'] = []
 +        if_count=0
 +        for if_id in node['interface_ids']:
 +            interface = Interface(interfaces[if_id])
 +            interface['ipv4'] = interface['ip']
 +            interface['component_id'] = PlXrn(auth=self.driver.hrn,
 +                                              interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
 +            # interfaces in the manifest need a client id
 +            if slice:
 +                interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
 +            rspec_node['interfaces'].append(interface)
 +            if_count+=1
 +        tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids'] if tag_id in node_tags]
 +        rspec_node['tags'] = tags
 +        return rspec_node
 +
 +    def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, \
 +                             pl_initscripts, sliver_allocations):
 +        # get the granularity in second for the reservation system
 +        grain = self.driver.shell.GetLeaseGranularity()
 +        rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain)
 +        # xxx how to retrieve site['login_base']
 +        rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
 +        # remove interfaces from manifest
 +        rspec_node['interfaces'] = []
 +        # add sliver info
 +        rspec_sliver = Sliver({'sliver_id': sliver['urn'],
 +                         'name': sliver['name'],
 +                         'type': 'plab-vserver',
 +                         'tags': []})
 +        rspec_node['sliver_id'] = rspec_sliver['sliver_id']
 +        rspec_node['client_id'] = sliver_allocations[sliver['urn']].client_id
 +        if sliver_allocations[sliver['urn']].component_id:
 +            rspec_node['component_id'] = sliver_allocations[sliver['urn']].component_id
 +        rspec_node['slivers'] = [rspec_sliver]
 +
 +        # slivers always provide the ssh service
 +        login = Login({'authentication': 'ssh-keys', 
 +                       'hostname': sliver['hostname'], 
 +                       'port':'22', 
 +                       'username': sliver['name'],
 +                       'login': sliver['name']
 +                      })
 +        service = ServicesElement({'login': login,
 +                            'services_user': sliver['services_user']})
 +        rspec_node['services'] = [service]    
 +        return rspec_node      
 +
 +    def get_slice_tags(self, slice):
 +        slice_tag_ids = []
 +        slice_tag_ids.extend(slice['slice_tag_ids'])
 +        tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids})
 +        # sorted by node_id
 +        tags_dict = defaultdict(list)
 +        for tag in tags:
 +            tags_dict[tag['node_id']] = tag
 +        return tags_dict
  
 -        filter = {}
 +    def get_slice_nodes(self, slice, options={}):
 +        nodes_dict = {}
 +        filter = {'peer_id': None}
          tags_filter = {}
 -        if slice and 'node_ids' in slice and slice['node_ids']:
 +        if slice and slice.get('node_ids'):
              filter['node_id'] = slice['node_ids']
 -            tags_filter=filter.copy()
 -
 -        geni_available = options.get('geni_available')    
 +        else:
 +            # there are no nodes to look up
 +            return nodes_dict
 +        tags_filter=filter.copy()
 +        geni_available = options.get('geni_available')
          if geni_available == True:
 -            filter['boot_state'] = 'boot'     
 -        
 -        filter.update({'peer_id': None})
 +            filter['boot_state'] = 'boot'
          nodes = self.driver.shell.GetNodes(filter)
 -        
 -        # get the granularity in second for the reservation system
 -        grain = self.driver.shell.GetLeaseGranularity()
 -       
 -        site_ids = []
 -        interface_ids = []
 -        tag_ids = []
 -        nodes_dict = {}
          for node in nodes:
 -            site_ids.append(node['site_id'])
 -            interface_ids.extend(node['interface_ids'])
 -            tag_ids.extend(node['node_tag_ids'])
              nodes_dict[node['node_id']] = node
 - 
 -        # get sites
 -        sites_dict  = self.get_sites({'site_id': site_ids}) 
 -        # get interfaces
 -        interfaces = self.get_interfaces({'interface_id':interface_ids}) 
 -        # get tags
 -        node_tags = self.get_node_tags(tags_filter)
 -        # get initscripts
 -        pl_initscripts = self.get_pl_initscripts()
 -        
 -        links = self.get_links(sites_dict, nodes_dict, interfaces)
 -
 -        rspec_nodes = []
 -        for node in nodes:
 -            # skip whitelisted nodes
 -            if node['slice_ids_whitelist']:
 -                if not slice or slice['slice_id'] not in node['slice_ids_whitelist']:
 -                    continue
 -            rspec_node = Node()
 -            # xxx how to retrieve site['login_base']
 -            site_id=node['site_id']
 -            site=sites_dict[site_id]
 -            rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
 -            rspec_node['component_name'] = node['hostname']
 -            rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
 -            rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
 -            # do not include boot state (<available> element) in the manifest rspec
 -            if not slice:     
 -                rspec_node['boot_state'] = node['boot_state']
 -
 -            #add the exclusive tag to distinguish between Shared and Reservable nodes
 -            if node['node_type'] == 'reservable':
 -                rspec_node['exclusive'] = 'true'
 +        return nodes_dict
 +
 +    def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}):
 +        if rspec_node['sliver_id'] in sliver_allocations:
 +            # set sliver allocation and operational status
 +            sliver_allocation = sliver_allocations[rspec_node['sliver_id']]
 +            if sliver_allocation:
 +                allocation_status = sliver_allocation.allocation_state
 +                if allocation_status == 'geni_allocated':
 +                    op_status =  'geni_pending_allocation'
 +                elif allocation_status == 'geni_provisioned':
 +                    if rspec_node['boot_state'] == 'boot':
 +                        op_status = 'geni_ready'
 +                    else:
 +                        op_status = 'geni_failed'
 +                else:
 +                    op_status = 'geni_unknown'
              else:
 -                rspec_node['exclusive'] = 'false'
 -
 -            rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
 -                                            HardwareType({'name': 'pc'})]
 -            # only doing this because protogeni rspec needs
 -            # to advertise available initscripts 
 -            rspec_node['pl_initscripts'] = pl_initscripts.values()
 -             # add site/interface info to nodes.
 -            # assumes that sites, interfaces and tags have already been prepared.
 -            site = sites_dict[node['site_id']]
 -            if site['longitude'] and site['latitude']:  
 -                location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
 -                rspec_node['location'] = location
 -            # Granularity
 -            granularity = Granularity({'grain': grain})
 -            rspec_node['granularity'] = granularity
 -
 -            rspec_node['interfaces'] = []
 -            if_count=0
 -            for if_id in node['interface_ids']:
 -                interface = Interface(interfaces[if_id]) 
 -                interface['ipv4'] = interface['ip']
 -                interface['component_id'] = PlXrn(auth=self.driver.hrn, 
 -                                                  interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
 -                # interfaces in the manifest need a client id
 -                if slice:
 -                    interface['client_id'] = "%s:%s" % (node['node_id'], if_id)            
 -                rspec_node['interfaces'].append(interface)
 -                if_count+=1
 -
 -            tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']\
 -                    if tag_id in node_tags]
 -            rspec_node['tags'] = tags
 -            if node['node_id'] in slivers:
 -                # add sliver info
 -                sliver = slivers[node['node_id']]
 -                rspec_node['sliver_id'] = sliver['sliver_id']
 -                rspec_node['slivers'] = [sliver]
 -                for tag in sliver['tags']:
 -                    if tag['tagname'] == 'client_id':
 -                         rspec_node['client_id'] = tag['value']
 -                
 -                # slivers always provide the ssh service
 -                login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']})
 -                service = Services({'login': login})
 -                rspec_node['services'] = [service]
 -            rspec_nodes.append(rspec_node)
 -        return (rspec_nodes, links)
 -             
 +                allocation_status = 'geni_unallocated'    
 +        # required fields
 +        geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
 +                       'geni_expires': rspec_node['expires'],
 +                       'geni_allocation_status' : allocation_status,
 +                       'geni_operational_status': op_status,
 +                       'geni_error': '',
 +                       }
 +        return geni_sliver        
  
-     def get_leases(self, slice=None, options={}):
+     def get_leases(self, slice_xrn=None, slice=None, options={}):
          
+         if slice_xrn and not slice:
+             return []
          now = int(time.time())
          filter={}
          filter.update({'clip':now})
              site_id=lease['site_id']
              site=sites_dict[site_id]
  
-             rspec_lease['lease_id'] = lease['lease_id']
-             rspec_lease['component_id'] = PlXrn(self.driver.hrn, hostname=lease['hostname']).urn
-             slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name'])
-             slice_urn = hrn_to_urn(slice_hrn, 'slice')
 -            #rspec_lease['lease_id'] = lease['lease_id']
+             rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname'])
+             if slice_xrn:
+                 slice_urn = slice_xrn
+                 slice_hrn = urn_to_hrn(slice_urn)
+             else:
+                 slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name'])
+                 slice_urn = hrn_to_urn(slice_hrn, 'slice')
              rspec_lease['slice_id'] = slice_urn
              rspec_lease['start_time'] = lease['t_from']
              rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain
  
          version_manager = VersionManager()
          version = version_manager.get_version(version)
 -        if not slice_xrn:
 -            rspec_version = version_manager._get_version(version.type, version.version, 'ad')
 -        else:
 -            rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
 -
 -        slice, slivers = self.get_slice_and_slivers(slice_xrn)
 +        rspec_version = version_manager._get_version(version.type, version.version, 'ad')
          rspec = RSpec(version=rspec_version, user_options=options)
 -        if slice and 'expires' in slice:
 -            rspec.xml.set('expires',  datetime_to_string(utcparse(slice['expires'])))
 -
 -        if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'leases':
 -            if slice_xrn and not slivers:
 -                nodes, links = [], []
 -            else:
 -                nodes, links = self.get_nodes_and_links(slice_xrn, slice, slivers, options)
 -            rspec.version.add_nodes(nodes)
 +       
 +        if not options.get('list_leases') or options['list_leases'] != 'leases':
 +            # get nodes
 +            nodes  = self.get_nodes(options)
 +            site_ids = []
 +            interface_ids = []
 +            tag_ids = []
 +            nodes_dict = {}
 +            for node in nodes:
 +                site_ids.append(node['site_id'])
 +                interface_ids.extend(node['interface_ids'])
 +                tag_ids.extend(node['node_tag_ids'])
 +                nodes_dict[node['node_id']] = node
 +            sites = self.get_sites({'site_id': site_ids})
 +            interfaces = self.get_interfaces({'interface_id':interface_ids})
 +            node_tags = self.get_node_tags({'node_tag_id': tag_ids})
 +            pl_initscripts = self.get_pl_initscripts()
 +            # convert nodes to rspec nodes
 +            rspec_nodes = []
 +            for node in nodes:
 +                rspec_node = self.node_to_rspec_node(node, sites, interfaces, node_tags, pl_initscripts)
 +                rspec_nodes.append(rspec_node)
 +            rspec.version.add_nodes(rspec_nodes)
 +
 +            # add links
 +            links = self.get_links(sites, nodes_dict, interfaces)        
              rspec.version.add_links(links)
 -            # add sliver defaults
 -            default_sliver = slivers.get(None, [])
 -            if default_sliver:
 -                default_sliver_attribs = default_sliver.get('tags', [])
 -                for attrib in default_sliver_attribs:
 -                    logger.info(attrib)
 -                    rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
 -        
++
+         if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources':
+            leases = self.get_leases(slice_xrn, slice)
+            rspec.version.add_leases(leases)
          return rspec.toxml()
  
 +    def describe(self, urns, version=None, options={}):
 +        version_manager = VersionManager()
 +        version = version_manager.get_version(version)
 +        rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
 +        rspec = RSpec(version=rspec_version, user_options=options)
 +
 +        # get slivers
 +        geni_slivers = []
 +        slivers = self.get_slivers(urns, options)
 +        if slivers:
 +            rspec_expires = datetime_to_string(utcparse(slivers[0]['expires']))
 +        else:
 +            rspec_expires = datetime_to_string(utcparse(time.time()))      
 +        rspec.xml.set('expires',  rspec_expires)
 +
 +        # lookup the sliver allocations
 +        geni_urn = urns[0]
 +        sliver_ids = [sliver['sliver_id'] for sliver in slivers]
 +        constraint = SliverAllocation.sliver_id.in_(sliver_ids)
 +        sliver_allocations = dbsession.query(SliverAllocation).filter(constraint)
 +        sliver_allocation_dict = {}
 +        for sliver_allocation in sliver_allocations:
 +            geni_urn = sliver_allocation.slice_urn
 +            sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
 +      
 +        if not options.get('list_leases') or options['list_leases'] != 'leases':
 +            # add slivers
 +            site_ids = []
 +            interface_ids = []
 +            tag_ids = []
 +            nodes_dict = {}
 +            for sliver in slivers:
 +                site_ids.append(sliver['site_id'])
 +                interface_ids.extend(sliver['interface_ids'])
 +                tag_ids.extend(sliver['node_tag_ids'])
 +                nodes_dict[sliver['node_id']] = sliver
 +            sites = self.get_sites({'site_id': site_ids})
 +            interfaces = self.get_interfaces({'interface_id':interface_ids})
 +            node_tags = self.get_node_tags({'node_tag_id': tag_ids})
 +            pl_initscripts = self.get_pl_initscripts()
 +            rspec_nodes = []
 +            for sliver in slivers:
 +                if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
 +                    continue
 +                rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, 
 +                                                       pl_initscripts, sliver_allocation_dict)
 +                # manifest node element shouldn't contain available attribute
 +                rspec_node.pop('available')
 +                rspec_nodes.append(rspec_node) 
 +                geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, sliver_allocation_dict)
 +                geni_slivers.append(geni_sliver)
 +            rspec.version.add_nodes(rspec_nodes)
 +
 +            # add sliver defaults
 +            #default_sliver = slivers.get(None, [])
 +            #if default_sliver:
 +            #    default_sliver_attribs = default_sliver.get('tags', [])
 +            #    for attrib in default_sliver_attribs:
 +            #        rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value'])
 +
 +            # add links 
 +            links = self.get_links(sites, nodes_dict, interfaces)        
 +            rspec.version.add_links(links)
 +
 +        if not options.get('list_leases') or options['list_leases'] != 'resources':
 +            if slivers:
 +                leases = self.get_leases(slivers[0])
 +                rspec.version.add_leases(leases)
  
 +               
 +        return {'geni_urn': geni_urn, 
 +                'geni_rspec': rspec.toxml(),
 +                'geni_slivers': geni_slivers}
@@@ -658,120 -745,89 +658,112 @@@ class PlDriver (Driver)
          slices.verify_slice_links(slice, rspec.version.get_link_requests(), nodes)
  
          # add/remove leases
-         requested_leases = []
-         kept_leases = []
-         for lease in rspec.version.get_leases():
-             requested_lease = {}
-             if not lease.get('lease_id'):
-                requested_lease['hostname'] = xrn_to_hostname(lease.get('component_id').strip())
-                requested_lease['start_time'] = lease.get('start_time')
-                requested_lease['duration'] = lease.get('duration')
-             else:
-                kept_leases.append(int(lease['lease_id']))
-             if requested_lease.get('hostname'):
-                 requested_leases.append(requested_lease)
+         try:
+            rspec_requested_leases = rspec.version.get_leases()
+            leases = slices.verify_slice_leases(slice, rspec_requested_leases, peer)
+         except:
+            pass
 -        #requested_leases = []
 -        #kept_leases = []
 -        #for lease in rspec.version.get_leases():
 -        #    requested_lease = {}
 -        #    if not lease.get('lease_id'):
 -        #       requested_lease['hostname'] = xrn_to_hostname(lease.get('component_id').strip())
 -        #       requested_lease['start_time'] = lease.get('start_time')
 -        #       requested_lease['duration'] = lease.get('duration')
 -        #    else:
 -        #       kept_leases.append(int(lease['lease_id']))
 -        #    if requested_lease.get('hostname'):
 -        #        requested_leases.append(requested_lease)
 -
 -        #leases = slices.verify_slice_leases(slice, requested_leases, kept_leases, peer)
 -    
 +
-         leases = slices.verify_slice_leases(slice, requested_leases, kept_leases, peer)
          # handle MyPLC peer association.
          # only used by plc and ple.
 -        slices.handle_peer(site, slice, persons, peer)
 +        slices.handle_peer(site, slice, None, peer)
          
 -        return aggregate.get_rspec(slice_xrn=slice_urn, 
 -                                   version=rspec.version)
 +        return aggregate.describe([xrn.get_urn()], version=rspec.version)
  
 -    def delete_sliver (self, slice_urn, slice_hrn, creds, options):
 -        slicename = hrn_to_pl_slicename(slice_hrn)
 -        slices = self.shell.GetSlices({'name': slicename})
 -        if not slices:
 -            return True
 -        slice = slices[0]
 -        
 -        # leases
 -        leases = self.shell.GetLeases({'name': slicename})
 -        leases_ids = [lease['lease_id'] for lease in leases ]
 -        # determine if this is a peer slice
 -        # xxx I wonder if this would not need to use PlSlices.get_peer instead 
 -        # in which case plc.peers could be deprecated as this here
 -        # is the only/last call to this last method in plc.peers
 -        peer = peers.get_peer(self, slice_hrn)
 -        try:
 -            if peer:
 -                self.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer)
 -            self.shell.DeleteSliceFromNodes(slicename, slice['node_ids'])
 -            if len(leases_ids) > 0:
 -                self.shell.DeleteLeases(leases_ids)
 -        finally:
 -            if peer:
 -                self.shell.BindObjectToPeer('slice', slice['slice_id'], peer, slice['peer_slice_id'])
 -        return True
 -    
 -    def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options):
 -        slicename = hrn_to_pl_slicename(slice_hrn)
 -        slices = self.shell.GetSlices({'name': slicename}, ['slice_id'])
 -        if not slices:
 -            raise RecordNotFound(slice_hrn)
 -        slice = slices[0]
 +    def provision(self, urns, options={}):
 +        # update users
 +        slices = PlSlices(self)
 +        aggregate = PlAggregate(self)
 +        slivers = aggregate.get_slivers(urns)
 +        slice = slivers[0]
 +        peer = slices.get_peer(slice['hrn'])
 +        sfa_peer = slices.get_sfa_peer(slice['hrn'])
 +        users = options.get('geni_users', [])
 +        persons = slices.verify_persons(None, slice, users, peer, sfa_peer, options=options)
 +        slices.handle_peer(None, None, persons, peer)
 +        # update sliver allocation states and set them to geni_provisioned
 +        sliver_ids = [sliver['sliver_id'] for sliver in slivers]
 +        SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned')
 +        version_manager = VersionManager()
 +        rspec_version = version_manager.get_version(options['geni_rspec_version']) 
 +        return self.describe(urns, rspec_version, options=options)
 +
 +    def delete(self, urns, options={}):
 +        # collect sliver ids so we can update sliver allocation states after
 +        # we remove the slivers.
 +        aggregate = PlAggregate(self)
 +        slivers = aggregate.get_slivers(urns)
 +        if slivers:
 +            slice_id = slivers[0]['slice_id'] 
 +            node_ids = []
 +            sliver_ids = []
 +            for sliver in slivers:
 +                node_ids.append(sliver['node_id'])
 +                sliver_ids.append(sliver['sliver_id']) 
 +
 +            # determine if this is a peer slice
 +            # xxx I wonder if this would not need to use PlSlices.get_peer instead 
 +            # in which case plc.peers could be deprecated as this here
 +            # is the only/last call to this last method in plc.peers
 +            slice_hrn = PlXrn(auth=self.hrn, slicename=slivers[0]['name']).get_hrn()     
 +            peer = peers.get_peer(self, slice_hrn)
 +            try:
 +                if peer:
 +                    self.shell.UnBindObjectFromPeer('slice', slice_id, peer)
 +     
 +                self.shell.DeleteSliceFromNodes(slice_id, node_ids)
 +     
 +                # delete sliver allocation states
 +                SliverAllocation.delete_allocations(sliver_ids)
 +            finally:
 +                if peer:
 +                    self.shell.BindObjectToPeer('slice', slice_id, peer, slice['peer_slice_id'])
 +
 +        # prepare return struct
 +        geni_slivers = []
 +        for sliver in slivers:
 +            geni_slivers.append(
 +                {'geni_sliver_urn': sliver['sliver_id'],
 +                 'geni_allocation_status': 'geni_unallocated',
 +                 'geni_expires': datetime_to_string(utcparse(sliver['expires']))})  
 +        return geni_slivers
-     
++
 +    def renew (self, urns, expiration_time, options={}):
 +        aggregate = PlAggregate(self)
 +        slivers = aggregate.get_slivers(urns)
 +        if not slivers:
 +            raise SearchFailed(urns)
 +        slice = slivers[0]
          requested_time = utcparse(expiration_time)
          record = {'expires': int(datetime_to_epoch(requested_time))}
 -        try:
 -            self.shell.UpdateSlice(slice['slice_id'], record)
 -            return True
 -        except:
 -            return False
 +        self.shell.UpdateSlice(slice['slice_id'], record)
 +        description = self.describe(urns, None, options)
 +        return description['geni_slivers']
 +            
  
 -    # remove the 'enabled' tag 
 -    def start_slice (self, slice_urn, slice_hrn, creds):
 -        slicename = hrn_to_pl_slicename(slice_hrn)
 -        slices = self.shell.GetSlices({'name': slicename}, ['slice_id'])
 -        if not slices:
 -            raise RecordNotFound(slice_hrn)
 -        slice_id = slices[0]['slice_id']
 -        slice_tags = self.shell.GetSliceTags({'slice_id': slice_id, 'tagname': 'enabled'}, ['slice_tag_id'])
 -        # just remove the tag if it exists
 -        if slice_tags:
 -            self.shell.DeleteSliceTag(slice_tags[0]['slice_tag_id'])
 -        return 1
 +    def perform_operational_action (self, urns, action, options={}):
 +        # MyPLC doesn't support operational actions. Lets pretend like it
 +        # supports start, but reject everything else.
 +        action = action.lower()
 +        if action not in ['geni_start']:
 +            raise UnsupportedOperation(action)
 +
 +        # fault if sliver is not full allocated (operational status is geni_pending_allocation)
 +        description = self.describe(urns, None, options)
 +        for sliver in description['geni_slivers']:
 +            if sliver['geni_operational_status'] == 'geni_pending_allocation':
 +                raise UnsupportedOperation(action, "Sliver must be fully allocated (operational status is not geni_pending_allocation)")
 +        #
 +        # Perform Operational Action Here
 +        #
 +
 +        geni_slivers = self.describe(urns, None, options)['geni_slivers']
 +        return geni_slivers
  
      # set the 'enabled' tag to 0
 -    def stop_slice (self, slice_urn, slice_hrn, creds):
 -        slicename = hrn_to_pl_slicename(slice_hrn)
 +    def shutdown (self, xrn, options={}):
 +        xrn = PlXrn(xrn=xrn, type='slice')
 +        slicename = xrn.pl_slicename()
          slices = self.shell.GetSlices({'name': slicename}, ['slice_id'])
          if not slices:
              raise RecordNotFound(slice_hrn)
@@@ -197,11 -221,27 +232,27 @@@ class PlSlices
          nodes = self.driver.shell.GetNodes(slice['node_ids'], ['node_id', 'hostname', 'interface_ids'])
          current_slivers = [node['hostname'] for node in nodes]
  
+         requested_slivers = []
+         tags = []
+         for node in slivers:
+             hostname = None
+             if node.get('component_name'):
+                 hostname = node.get('component_name').strip()
+             elif node.get('component_id'):
+                 hostname = xrn_to_hostname(node.get('component_id').strip())
+             if node.get('client_id'):
+                 tags.append({'slicename': slice['name'], 
+                              'tagname': 'client_id',
+                              'value': node['client_id'],
+                              'node': hostname})
+             if hostname:
+                 requested_slivers.append(hostname)
+         
          # remove nodes not in rspec
 -        deleted_nodes = list(set(current_slivers).difference(requested_slivers))
 +        deleted_nodes = list(set(current_slivers).difference(slivers.keys()))
  
          # add nodes from rspec
 -        added_nodes = list(set(requested_slivers).difference(current_slivers))        
 +        added_nodes = list(set(slivers.keys()).difference(current_slivers))        
  
          try:
              if peer:
@@@ -1,8 -1,9 +1,9 @@@
  from sfa.rspecs.elements.element import Element
   
 -class Node(Element):
 +class NodeElement(Element):
      
      fields = [
+         'client_id',
          'component_id',
          'component_name',
          'component_manager_id',
Simple merge
Simple merge
Simple merge
Simple merge