cosmetic changes in sendmail.py - please the linter

[plcapi.git] / PLC / Methods / GetSlivers.py

diff --git a/PLC/Methods/GetSlivers.py b/PLC/Methods/GetSlivers.py
index 9f86615..1a0349f 100644 (file)
--- a/PLC/Methods/GetSlivers.py
+++ b/PLC/Methods/GetSlivers.py
@@ -1,5 +1,3 @@
-# $Id$
-# $URL$

 import time
 
 from PLC.Faults import *
 import time
 
 from PLC.Faults import *


@@ -21,13 +19,16 @@ from PLC.InitScripts import InitScript, InitScripts
 from PLC.Leases import Lease, Leases
 from PLC.Timestamp import Duration
 from PLC.Methods.GetSliceFamily import GetSliceFamily
 from PLC.Leases import Lease, Leases
 from PLC.Timestamp import Duration
 from PLC.Methods.GetSliceFamily import GetSliceFamily

+from PLC.PersonTags import PersonTag,PersonTags

 
 from PLC.Accessors.Accessors_standard import *
 
 from PLC.Accessors.Accessors_standard import *

+from functools import reduce

 
 # XXX used to check if slice expiration time is sane
 
 # XXX used to check if slice expiration time is sane

-MAXINT =  2L**31-1
+MAXINT =  2**31-1

 
 

-def get_slivers(api, auth, slice_filter, node = None):
+# slice_filter essentially contains the slice_ids for the relevant slices (on the node + system & delegated slices)
+def get_slivers(api, caller, auth, slice_filter, node = None):

     # Get slice information
     slices = Slices(api, slice_filter, ['slice_id', 'name', 'instantiation', 'expires', 'person_ids', 'slice_tag_ids'])
 
     # Get slice information
     slices = Slices(api, slice_filter, ['slice_id', 'name', 'instantiation', 'expires', 'person_ids', 'slice_tag_ids'])
 


@@ -43,7 +44,7 @@ def get_slivers(api, auth, slice_filter, node = None):
 
     # Build up list of keys
     key_ids = set()
 
     # Build up list of keys
     key_ids = set()

-    for person in all_persons.values():
+    for person in list(all_persons.values()):

         key_ids.update(person['key_ids'])
 
     # Get user account keys
         key_ids.update(person['key_ids'])
 
     # Get user account keys


@@ -77,7 +78,7 @@ def get_slivers(api, auth, slice_filter, node = None):
         # Per-node sliver attributes take precedence over global
         # slice attributes, so set them first.
         # Then comes nodegroup slice attributes
         # Per-node sliver attributes take precedence over global
         # slice attributes, so set them first.
         # Then comes nodegroup slice attributes

-       # Followed by global slice attributes
+        # Followed by global slice attributes

         sliver_attributes = []
 
         if node is not None:
         sliver_attributes = []
 
         if node is not None:


@@ -86,16 +87,17 @@ def get_slivers(api, auth, slice_filter, node = None):
                 attributes.append({'tagname': sliver_attribute['tagname'],
                                    'value': sliver_attribute['value']})
 
                 attributes.append({'tagname': sliver_attribute['tagname'],
                                    'value': sliver_attribute['value']})
 

-           # set nodegroup slice attributes
-           for slice_tag in [ a for a in slice_tags if a['nodegroup_id'] in node['nodegroup_ids'] ]:
-               # Do not set any nodegroup slice attributes for
+            # set nodegroup slice attributes
+            for slice_tag in [ a for a in slice_tags if a['nodegroup_id'] in node['nodegroup_ids'] ]:
+                # Do not set any nodegroup slice attributes for

                 # which there is at least one sliver attribute
                 # already set.
                 # which there is at least one sliver attribute
                 # already set.

-               if slice_tag not in slice_tags:
-                   attributes.append({'tagname': slice_tag['tagname'],
-                                  'value': slice_tag['value']})
+                if slice_tag['tagname'] not in sliver_attributes:
+                    sliver_attributes.append(slice_tag['tagname'])
+                    attributes.append({'tagname': slice_tag['tagname'],
+                                       'value': slice_tag['value']})

 
 

-        for slice_tag in [ a for a in slice_tags if a['node_id'] is None ]:
+        for slice_tag in [ a for a in slice_tags if a['node_id'] is None and a['nodegroup_id'] is None ]:

             # Do not set any global slice attributes for
             # which there is at least one sliver attribute
             # already set.
             # Do not set any global slice attributes for
             # which there is at least one sliver attribute
             # already set.


@@ -108,7 +110,7 @@ def get_slivers(api, auth, slice_filter, node = None):
         if slice['expires'] > MAXINT:  slice['expires']= MAXINT
 
         # expose the slice vref as computed by GetSliceFamily
         if slice['expires'] > MAXINT:  slice['expires']= MAXINT
 
         # expose the slice vref as computed by GetSliceFamily

-        family = GetSliceFamily (api).call(auth, slice['slice_id'])
+        family = GetSliceFamily (api,caller).call(auth, slice['slice_id'])

 
         slivers.append({
             'name': slice['name'],
 
         slivers.append({
             'name': slice['name'],


@@ -122,6 +124,24 @@ def get_slivers(api, auth, slice_filter, node = None):
 
     return slivers
 
 
     return slivers
 

+### The pickle module, used in conjunction with caching has a restriction that it does not
+### work on "connection objects." It doesn't matter if the connection object has
+### an 'str' or 'repr' method, there is a taint check that throws an exception if
+### the pickled class is found to derive from a connection.
+### (To be moved to Method.py)
+
+def sanitize_for_pickle (obj):
+    if (isinstance(obj, dict)):
+        parent = dict(obj)
+        for k in list(parent.keys()): parent[k] = sanitize_for_pickle (parent[k])
+        return parent
+    elif (isinstance(obj, list)):
+        parent = list(obj)
+        parent = list(map(sanitize_for_pickle, parent))
+        return parent
+    else:
+        return obj
+

 class GetSlivers(Method):
     """
     Returns a struct containing information about the specified node
 class GetSlivers(Method):
     """
     Returns a struct containing information about the specified node


@@ -149,7 +169,7 @@ class GetSlivers(Method):
         'interfaces': [Interface.fields],
         'groups': [NodeGroup.fields['groupname']],
         'conf_files': [ConfFile.fields],
         'interfaces': [Interface.fields],
         'groups': [NodeGroup.fields['groupname']],
         'conf_files': [ConfFile.fields],

-       'initscripts': [InitScript.fields],
+        'initscripts': [InitScript.fields],

         'accounts': [{
             'name': Parameter(str, "unix style account name", max = 254),
             'keys': [{
         'accounts': [{
             'name': Parameter(str, "unix style account name", max = 254),
             'keys': [{


@@ -188,6 +208,10 @@ class GetSlivers(Method):
     }
 
     def call(self, auth, node_id_or_hostname = None):
     }
 
     def call(self, auth, node_id_or_hostname = None):

+        return self.raw_call(auth, node_id_or_hostname)
+
+
+    def raw_call(self, auth, node_id_or_hostname):

         timestamp = int(time.time())
 
         # Get node
         timestamp = int(time.time())
 
         # Get node


@@ -195,22 +219,22 @@ class GetSlivers(Method):
             if isinstance(self.caller, Node):
                 node = self.caller
             else:
             if isinstance(self.caller, Node):
                 node = self.caller
             else:

-                raise PLCInvalidArgument, "'node_id_or_hostname' not specified"
+                raise PLCInvalidArgument("'node_id_or_hostname' not specified")

         else:
             nodes = Nodes(self.api, [node_id_or_hostname])
             if not nodes:
         else:
             nodes = Nodes(self.api, [node_id_or_hostname])
             if not nodes:

-                raise PLCInvalidArgument, "No such node"
+                raise PLCInvalidArgument("No such node")

             node = nodes[0]
 
             if node['peer_id'] is not None:
             node = nodes[0]
 
             if node['peer_id'] is not None:

-                raise PLCInvalidArgument, "Not a local node"
+                raise PLCInvalidArgument("Not a local node")

 
         # Get interface information
         interfaces = Interfaces(self.api, node['interface_ids'])
 
         # Get node group information
         nodegroups = NodeGroups(self.api, node['nodegroup_ids']).dict('groupname')
 
         # Get interface information
         interfaces = Interfaces(self.api, node['interface_ids'])
 
         # Get node group information
         nodegroups = NodeGroups(self.api, node['nodegroup_ids']).dict('groupname')

-        groups = nodegroups.keys()
+        groups = list(nodegroups.keys())

 
         # Get all (enabled) configuration files
         all_conf_files = ConfFiles(self.api, {'enabled': True}).dict()
 
         # Get all (enabled) configuration files
         all_conf_files = ConfFiles(self.api, {'enabled': True}).dict()


@@ -219,41 +243,41 @@ class GetSlivers(Method):
         # Global configuration files are the default. If multiple
         # entries for the same global configuration file exist, it is
         # undefined which one takes precedence.
         # Global configuration files are the default. If multiple
         # entries for the same global configuration file exist, it is
         # undefined which one takes precedence.

-        for conf_file in all_conf_files.values():
+        for conf_file in list(all_conf_files.values()):

             if not conf_file['node_ids'] and not conf_file['nodegroup_ids']:
                 conf_files[conf_file['dest']] = conf_file
             if not conf_file['node_ids'] and not conf_file['nodegroup_ids']:
                 conf_files[conf_file['dest']] = conf_file

-        
+

         # Node group configuration files take precedence over global
         # ones. If a node belongs to multiple node groups for which
         # the same configuration file is defined, it is undefined
         # which one takes precedence.
         # Node group configuration files take precedence over global
         # ones. If a node belongs to multiple node groups for which
         # the same configuration file is defined, it is undefined
         # which one takes precedence.

-        for nodegroup in nodegroups.values():
+        for nodegroup in list(nodegroups.values()):

             for conf_file_id in nodegroup['conf_file_ids']:
                 if conf_file_id in all_conf_files:
                     conf_file = all_conf_files[conf_file_id]
                     conf_files[conf_file['dest']] = conf_file
             for conf_file_id in nodegroup['conf_file_ids']:
                 if conf_file_id in all_conf_files:
                     conf_file = all_conf_files[conf_file_id]
                     conf_files[conf_file['dest']] = conf_file

-        
+

         # Node configuration files take precedence over node group
         # configuration files.
         for conf_file_id in node['conf_file_ids']:
             if conf_file_id in all_conf_files:
                 conf_file = all_conf_files[conf_file_id]
         # Node configuration files take precedence over node group
         # configuration files.
         for conf_file_id in node['conf_file_ids']:
             if conf_file_id in all_conf_files:
                 conf_file = all_conf_files[conf_file_id]

-                conf_files[conf_file['dest']] = conf_file            
+                conf_files[conf_file['dest']] = conf_file

 
 

-       # Get all (enabled) initscripts
-       initscripts = InitScripts(self.api, {'enabled': True})  
+        # Get all (enabled) initscripts
+        initscripts = InitScripts(self.api, {'enabled': True})

 
         # Get system slices
         system_slice_tags = SliceTags(self.api, {'tagname': 'system', 'value': '1'}).dict('slice_id')
 
         # Get system slices
         system_slice_tags = SliceTags(self.api, {'tagname': 'system', 'value': '1'}).dict('slice_id')

-        system_slice_ids = system_slice_tags.keys()
-       
-       # Get nm-controller slices
+        system_slice_ids = list(system_slice_tags.keys())
+
+        # Get nm-controller slices

         # xxx Thierry: should these really be exposed regardless of their mapping to nodes ?
         # xxx Thierry: should these really be exposed regardless of their mapping to nodes ?

-       controller_and_delegated_slices = Slices(self.api, {'instantiation': ['nm-controller', 'delegated']}, ['slice_id']).dict('slice_id')
-       controller_and_delegated_slice_ids = controller_and_delegated_slices.keys()
-       slice_ids = system_slice_ids + controller_and_delegated_slice_ids + node['slice_ids']
+        controller_and_delegated_slices = Slices(self.api, {'instantiation': ['nm-controller', 'delegated']}, ['slice_id']).dict('slice_id')
+        controller_and_delegated_slice_ids = list(controller_and_delegated_slices.keys())
+        slice_ids = system_slice_ids + controller_and_delegated_slice_ids + node['slice_ids']

 
 

-       slivers = get_slivers(self.api, auth, slice_ids, node)
+        slivers = get_slivers(self.api, self.caller, auth, slice_ids, node)

 
         # get the special accounts and keys needed for the node
         # root
 
         # get the special accounts and keys needed for the node
         # root


@@ -267,22 +291,37 @@ class GetSlivers(Method):
         # reduce ( reduce_flatten_list, [ [1] , [2,3] ], []) => [ 1,2,3 ]
         def reduce_flatten_list (x,y): return x+y
 
         # reduce ( reduce_flatten_list, [ [1] , [2,3] ], []) => [ 1,2,3 ]
         def reduce_flatten_list (x,y): return x+y
 

+        # root users are users marked with the tag 'isrootonsite'. Hack for Mlab and other sites in which admins participate in diagnosing problems.
+        def get_site_root_user_keys(api,site_id_or_name):
+           site = Sites (api,site_id_or_name,['person_ids'])[0]
+           all_site_persons = site['person_ids']
+           all_site_person_tags = PersonTags(self.api,{'person_id':all_site_persons,'tagname':'isrootonsite'},['value','person_id'])
+           site_root_person_tags = [r for r in all_site_person_tags if r['value']=='true']
+           site_root_person_ids = [r['person_id'] for r in site_root_person_tags]
+           key_ids = reduce (reduce_flatten_list,
+                             [ p['key_ids'] for p in \
+                                   Persons(api,{ 'person_id':site_root_person_ids,
+                                                 'enabled':True, '|role_ids' : [20, 40] },
+                                           ['key_ids']) ],
+                             [])
+           return [ key['key'] for key in Keys (api, key_ids) if key['key_type']=='ssh']
+

         # power users are pis and techs
         def get_site_power_user_keys(api,site_id_or_name):
             site = Sites (api,site_id_or_name,['person_ids'])[0]
         # power users are pis and techs
         def get_site_power_user_keys(api,site_id_or_name):
             site = Sites (api,site_id_or_name,['person_ids'])[0]

-            key_ids = reduce (reduce_flatten_list, 
+            key_ids = reduce (reduce_flatten_list,

                               [ p['key_ids'] for p in \
                               [ p['key_ids'] for p in \

-                                    Persons(api,{ 'person_id':site['person_ids'], 
-                                                  'enabled':True, '|role_ids' : [20, 40] }, 
+                                    Persons(api,{ 'person_id':site['person_ids'],
+                                                  'enabled':True, '|role_ids' : [20, 40] },

                                             ['key_ids']) ],
                               [])
             return [ key['key'] for key in Keys (api, key_ids) if key['key_type']=='ssh']
 
         # all admins regardless of their site
         def get_all_admin_keys(api):
                                             ['key_ids']) ],
                               [])
             return [ key['key'] for key in Keys (api, key_ids) if key['key_type']=='ssh']
 
         # all admins regardless of their site
         def get_all_admin_keys(api):

-            key_ids = reduce (reduce_flatten_list, 
+            key_ids = reduce (reduce_flatten_list,

                               [ p['key_ids'] for p in \
                               [ p['key_ids'] for p in \

-                                    Persons(api, {'peer_id':None, 'enabled':True, '|role_ids':[10] }, 
+                                    Persons(api, {'peer_id':None, 'enabled':True, '|role_ids':[10] },

                                             ['key_ids']) ],
                               [])
             return [ key['key'] for key in Keys (api, key_ids) if key['key_type']=='ssh']
                                             ['key_ids']) ],
                               [])
             return [ key['key'] for key in Keys (api, key_ids) if key['key_type']=='ssh']


@@ -291,49 +330,49 @@ class GetSlivers(Method):
         personsitekeys=get_site_power_user_keys(self.api,node['site_id'])
         accounts.append({'name':'site_admin','keys':personsitekeys})
 
         personsitekeys=get_site_power_user_keys(self.api,node['site_id'])
         accounts.append({'name':'site_admin','keys':personsitekeys})
 

-        # 'root' account setup on nodes from all 'admin' users
+        # 'root' account setup on nodes from all 'admin' users and ones marked with 'isrootonsite' for this site
+        siterootkeys=get_site_root_user_keys(self.api,node['site_id'])

         personsitekeys=get_all_admin_keys(self.api)
         personsitekeys=get_all_admin_keys(self.api)

+        personsitekeys.extend(siterootkeys)
+

         accounts.append({'name':'root','keys':personsitekeys})
 
         accounts.append({'name':'root','keys':personsitekeys})
 

-        hrn = GetNodeHrn(self.api).call(auth,node['node_id'])
+        hrn = GetNodeHrn(self.api,self.caller).call(auth,node['node_id'])

 
         # XMPP config for omf federation
         try:
             if not self.api.config.PLC_OMF_ENABLED:
 
         # XMPP config for omf federation
         try:
             if not self.api.config.PLC_OMF_ENABLED:

-                raise Exception,"OMF disabled"
-            xmpp={'server':self.api.config.PLC_OMF_XMPP_SERVER,
-                  'user':self.api.config.PLC_OMF_XMPP_USER,
-                  'password':self.api.config.PLC_OMF_XMPP_PASSWORD,
-                  }
+                raise Exception("OMF not enabled")
+            xmpp={'server':self.api.config.PLC_OMF_XMPP_SERVER}

         except:
         except:

-            xmpp={'server':None,'user':None,'password':None}
+            xmpp={'server':None}

 
 

-       node.update_last_contact()
+        node.update_last_contact()

 
         # expose leases & reservation policy
         # in a first implementation we only support none and lease_or_idle
 
         # expose leases & reservation policy
         # in a first implementation we only support none and lease_or_idle

-        lease_exposed_fields = [ 'slice_id', 't_from', 't_until', ]
+        lease_exposed_fields = [ 'slice_id', 't_from', 't_until', 'name', ]

         leases=None
         if node['node_type'] != 'reservable':
             reservation_policy='none'
         else:
             reservation_policy='lease_or_idle'
             # expose the leases for the next 24 hours
         leases=None
         if node['node_type'] != 'reservable':
             reservation_policy='none'
         else:
             reservation_policy='lease_or_idle'
             # expose the leases for the next 24 hours

-            leases = [ dict ( [ (k,l[k]) for k in lease_exposed_fields ] ) 
+            leases = [ dict ( [ (k,l[k]) for k in lease_exposed_fields ] )

                        for l in Leases (self.api, {'node_id':node['node_id'],
                                                    'clip': (timestamp, timestamp+24*Duration.HOUR),
                                                    '-SORT': 't_from',
                                                    }) ]
                        for l in Leases (self.api, {'node_id':node['node_id'],
                                                    'clip': (timestamp, timestamp+24*Duration.HOUR),
                                                    '-SORT': 't_from',
                                                    }) ]

-        granularity=Lease.granularity
+        granularity=self.api.config.PLC_RESERVATION_GRANULARITY

 
 

-        return {
+        raw_data = {

             'timestamp': timestamp,
             'node_id': node['node_id'],
             'hostname': node['hostname'],
             'interfaces': interfaces,
             'groups': groups,
             'timestamp': timestamp,
             'node_id': node['node_id'],
             'hostname': node['hostname'],
             'interfaces': interfaces,
             'groups': groups,

-            'conf_files': conf_files.values(),
-           'initscripts': initscripts,
+            'conf_files': list(conf_files.values()),
+            'initscripts': initscripts,

             'slivers': slivers,
             'accounts': accounts,
             'xmpp':xmpp,
             'slivers': slivers,
             'accounts': accounts,
             'xmpp':xmpp,


@@ -341,4 +380,8 @@ class GetSlivers(Method):
             'reservation_policy': reservation_policy,
             'leases':leases,
             'lease_granularity': granularity,
             'reservation_policy': reservation_policy,
             'leases':leases,
             'lease_granularity': granularity,

-            }
+        }
+
+        sanitized_data = sanitize_for_pickle (raw_data)
+        return sanitized_data
+