X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=system%2FTestPlc.py;h=7befc3f89d98a3dda0d2639885a1342b0003ab34;hb=510d4f034ed4bd4f8abb00cf04abcfbe53ab3ccd;hp=1ecfef0ae9ee7be4a0b89ea2906f8c3113b4457f;hpb=dbc34687d09c323f068217fab584b849c7759210;p=tests.git diff --git a/system/TestPlc.py b/system/TestPlc.py index 1ecfef0..7befc3f 100644 --- a/system/TestPlc.py +++ b/system/TestPlc.py @@ -152,65 +152,70 @@ class TestPlc: default_steps = [ 'show', SEP, - 'plcvm_delete','plcvm_timestamp','plcvm_create', SEP, + 'plcvm_delete', 'plcvm_timestamp', 'plcvm_create', SEP, 'django_install', 'plc_install', 'plc_configure', 'plc_start', SEP, 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP, - 'plcapi_urls','speed_up_slices', SEP, + 'plcapi_urls', 'speed_up_slices', SEP, 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP, -# slices created under plcsh interactively seem to be fine but these ones don't have the tags -# keep this our of the way for now - 'check_vsys_defaults_ignore', SEP, -# run this first off so it's easier to re-run on another qemu box - 'qemu_kill_mine', 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP, - 'qemu_clean_mine', 'qemu_export', 'qemu_cleanlog', SEP, - 'qemu_start', 'qemu_timestamp', 'qemu_nodefamily', SEP, - 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA, - 'sfi_configure@1', 'sfa_register_site@1','sfa_register_pi@1', SEPSFA, - 'sfa_register_user@1', 'sfa_update_user@1', 'sfa_register_slice@1', 'sfa_renew_slice@1', SEPSFA, - 'sfa_remove_user_from_slice@1','sfi_show_slice_researchers@1', - 'sfa_insert_user_in_slice@1','sfi_show_slice_researchers@1', SEPSFA, +# ss # slices created under plcsh interactively seem to be fine but these ones don't have the tags +# ss # keep this out of the way for now +# ss 'check_vsys_defaults_ignore', SEP, +# ss # run this first off so it's easier to re-run on another qemu box +# ss 'qemu_kill_mine', 'nodestate_reinstall', 'qemu_local_init', +# ss 'bootcd', 'qemu_local_config', SEP, +# ss 'qemu_clean_mine', 'qemu_export', 'qemu_cleanlog', SEP, +# ss 'qemu_start', 'qemu_timestamp', 'qemu_nodefamily', SEP, + 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', + 'sfa_start', 'sfa_import', SEPSFA, + 'sfi_configure@1', 'sfa_register_site@1', 'sfa_register_pi@1', SEPSFA, + 'sfa_register_user@1', 'sfa_update_user@1', + 'sfa_register_slice@1', 'sfa_renew_slice@1', SEPSFA, + 'sfa_remove_user_from_slice@1', 'sfi_show_slice_researchers@1', + 'sfa_insert_user_in_slice@1', 'sfi_show_slice_researchers@1', SEPSFA, 'sfa_discover@1', 'sfa_rspec@1', SEPSFA, 'sfa_allocate@1', 'sfa_provision@1', 'sfa_describe@1', SEPSFA, 'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA, 'sfi_list@1', 'sfi_show_site@1', 'sfa_utest@1', SEPSFA, # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot # but as the stress test might take a while, we sometimes missed the debug mode.. - 'probe_kvm_iptables', - 'ping_node', 'ssh_node_debug', 'plcsh_stress_test@1', SEP, - 'ssh_node_boot', 'node_bmlogs', 'ssh_slice', 'ssh_slice_basics', SEP, - 'ssh_slice_sfa@1', SEPSFA, - 'sfa_rspec_empty@1', 'sfa_allocate_empty@1', 'sfa_provision_empty@1','sfa_check_slice_plc_empty@1', SEPSFA, +# ss 'probe_kvm_iptables', +# ss 'ping_node', 'ssh_node_debug', 'plcsh_stress_test@1', SEP, +# ss 'ssh_node_boot', 'node_bmlogs', 'ssh_slice', 'ssh_slice_basics', SEP, +# ss 'ssh_slice_sfa@1', SEPSFA, + 'sfa_rspec_empty@1', 'sfa_allocate_empty@1', 'sfa_provision_empty@1', + 'sfa_check_slice_plc_empty@1', SEPSFA, 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA, - 'check_system_slice', SEP, +# ss 'check_system_slice', SEP, # for inspecting the slice while it runs the first time #'fail', # check slices are turned off properly - 'debug_nodemanager', - 'empty_slices', 'ssh_slice_off', 'slice_fs_deleted_ignore', SEP, - # check they are properly re-created with the same name - 'fill_slices', 'ssh_slice_again', SEP, +# ss 'debug_nodemanager', +# ss 'empty_slices', 'ssh_slice_off', 'slice_fs_deleted_ignore', SEP, +# ss # check they are properly re-created with the same name +# ss 'fill_slices', 'ssh_slice_again', SEP, 'gather_logs_force', SEP, ] other_steps = [ 'export', 'show_boxes', 'super_speed_up_slices', SEP, 'check_hooks', 'plc_stop', 'plcvm_start', 'plcvm_stop', SEP, - 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP, + 'delete_initscripts', 'delete_nodegroups', 'delete_all_sites', SEP, 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP, 'delete_leases', 'list_leases', SEP, 'populate', SEP, - 'nodestate_show','nodestate_safeboot','nodestate_boot', 'nodestate_upgrade', SEP, - 'nodedistro_show','nodedistro_f14','nodedistro_f18', SEP, - 'nodedistro_f20', 'nodedistro_f21','nodedistro_f22', SEP, + 'nodestate_show', 'nodestate_safeboot', 'nodestate_boot', 'nodestate_upgrade', SEP, + 'nodedistro_show', 'nodedistro_f14', 'nodedistro_f18', SEP, + 'nodedistro_f20', 'nodedistro_f21', 'nodedistro_f22', SEP, 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP, - 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA, - 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA, + 'sfa_install_core', 'sfa_install_sfatables', + 'sfa_install_plc', 'sfa_install_client', SEPSFA, + 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop', 'sfa_uninstall', 'sfi_clean', SEPSFA, 'sfa_get_expires', SEPSFA, 'plc_db_dump', 'plc_db_restore', SEP, 'check_netflow', 'check_drl', SEP, # used to be part of default steps but won't work since f27 'cross_check_tcp@1', 'slice_fs_present', 'check_initscripts', SEP, - 'standby_1_through_20','yes','no',SEP, + 'standby_1_through_20', 'yes', 'no', SEP, 'install_syslinux6', 'bonding_builds', 'bonding_nodes', SEP, ] default_bonding_steps = [ @@ -353,8 +358,8 @@ class TestPlc: # self.run_in_guest("yum-complete-transaction -y") return self.dnf_check_installed(rpms) - def pip_install(self, package): - return self.run_in_guest("pip install {}".format(package)) == 0 + def pip3_install(self, package): + return self.run_in_guest("pip3 install {}".format(package)) == 0 def auth_root(self): return {'Username' : self.plc_spec['settings']['PLC_ROOT_USER'], @@ -575,7 +580,7 @@ class TestPlc: continue if k == 'nodes': if v: - print('+ ','nodes : ', end=' ') + print('+ ', 'nodes : ', end=' ') for node in v: print(node['node_fields']['hostname'],'', end=' ') print('') @@ -615,7 +620,7 @@ class TestPlc: print(username,'', end=' ') print('') elif k == 'slice_fields': - print('+ fields',':', end=' ') + print('+ fields', ':', end=' ') print('max_nodes=',v['max_nodes'], end=' ') print('') else: @@ -724,7 +729,7 @@ class TestPlc: """ pip install Django """ - return self.pip_install('Django') + return self.pip3_install('Django') ### install_rpm def plc_install(self): @@ -747,8 +752,8 @@ class TestPlc: pkgs_list = [] pkgs_list.append("myplc") - pkgs_list.append("slicerepo-{}".format(nodefamily)) - pkgs_list.append("noderepo-{}".format(nodefamily)) + # pkgs_list.append("slicerepo-{}".format(nodefamily)) + # pkgs_list.append("noderepo-{}".format(nodefamily)) pkgs_string=" ".join(pkgs_list) return self.dnf_install(pkgs_list) @@ -883,7 +888,7 @@ class TestPlc: def delete_all_sites(self): "Delete all sites in PLC, and related objects" print('auth_root', self.auth_root()) - sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base']) + sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id', 'login_base']) for site in sites: # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore if site['login_base'] == self.plc_spec['settings']['PLC_SLICE_PREFIX']: @@ -1514,11 +1519,70 @@ class TestPlc: # in particular runs with --preserve (dont cleanup) and without --check # also it gets run twice, once with the --foreign option for creating fake foreign entries + def install_pip2(self): + + # xxx could make sense to mirror this one + + replacements = [ + "https://acc.dl.osdn.jp/storage/g/u/un/unitedrpms/32/x86_64/python2-pip-19.1.1-7.fc32.noarch.rpm", + ] + + return ( + self.run_in_guest("pip2 --version") == 0 + or self.run_in_guest("dnf install python2-pip") == 0 + or self.run_in_guest("dnf localinstall -y " + " ".join(replacements)) == 0) + + + def install_m2crypto(self): + + # installing m2crypto for python2 is increasingly difficult + # f29 and f31: dnf install python2-m2crypto + # f33: no longer available but the f31 repos below do the job just fine + # note that using pip2 does not look like a viable option because it does + # an install from sources and that's quite awkward + + replacements = [ + "http://mirror.onelab.eu/fedora/releases/31/Everything/x86_64/os/Packages/p/python2-typing-3.6.2-5.fc31.noarch.rpm", + "http://mirror.onelab.eu/fedora/releases/31/Everything/x86_64/os/Packages/p/python2-m2crypto-0.35.2-2.fc31.x86_64.rpm", + ] + + return ( + self.run_in_guest('python2 -c "import M2Crypto"', backslash=True) == 0 + or self.run_in_guest("pip2 install python2-m2crypto") == 0 + or self.run_in_guest("dnf localinstall -y " + " ".join(replacements)) == 0) + + # about pip2: + # we can try and use + # that qould then need to be mirrored + # so the logic goes like this + # check for pip2 command + # if not, try dnf install python2-pip + # if still not, dnf localinstall the above + + def sfa_install_all(self): "yum install sfa sfa-plc sfa-sfatables sfa-client" - return (self.dnf_install("sfa sfa-plc sfa-sfatables sfa-client") and - self.run_in_guest("systemctl enable sfa-registry")==0 and - self.run_in_guest("systemctl enable sfa-aggregate")==0) + + # the rpm/dnf packages named in python2-* are getting deprecated + # we use pip2 instead + # but that's not good for m2crypto + + pip_dependencies = [ + 'sqlalchemy-migrate', + 'lxml', + 'python-dateutil', + 'psycopg2-binary', + 'pyOpenSSL', + ] + + return ( + self.install_pip2() + and self.install_m2crypto() + and all((self.run_in_guest(f"pip2 install {dep}") == 0) + for dep in pip_dependencies) + and self.dnf_install("sfa sfa-plc sfa-sfatables sfa-client") + and self.run_in_guest("systemctl enable sfa-registry")==0 + and self.run_in_guest("systemctl enable sfa-aggregate")==0) def sfa_install_core(self): "yum install sfa" @@ -1588,7 +1652,7 @@ class TestPlc: except: print("Site {} already absent from PLC db".format(login_base)) - for spec_name in ['pi_spec','user_spec']: + for spec_name in ['pi_spec', 'user_spec']: user_spec = auth_sfa_spec[spec_name] username = user_spec['email'] try: @@ -1706,9 +1770,10 @@ class TestPlc: return self.run_in_guest('sfaadmin reg import_registry') == 0 def sfa_start(self): - "start SFA through systemctl" - return (self.start_stop_systemd('sfa-registry', 'start') and - self.start_stop_systemd('sfa-aggregate', 'start')) + "start SFA through systemctl - also install dependencies" + + return (self.start_stop_systemd('sfa-registry', 'start') + and self.start_stop_systemd('sfa-aggregate', 'start')) def sfi_configure(self): @@ -1947,7 +2012,7 @@ class TestPlc: if '@' in step: step, qualifier = step.split('@') # or be defined as forced or ignored by default - for keyword in ['_ignore','_force']: + for keyword in ['_ignore', '_force']: if step.endswith(keyword): step=step.replace(keyword,'') if step == SEP or step == SEPSFA :