defines persons_per_slice to avoid quadratic complexity - can run with DB dumps

diff --git a/TestPeers.py b/TestPeers.py
index b31c193..f6a4733 100755 (executable)
--- a/TestPeers.py
+++ b/TestPeers.py
@@ -74,17 +74,20 @@ def total_slivers ():
 
 ####################
 # set initial conditions
+# actual persons_per_slice is min(number_persons,number_persons_per_slice)
 # actual nodes_per_slice is min(number_nodes,number_nodes_per_slice)
 # this is to prevent quadractic test times on big tests
-def define_test (keys,sites,persons,nodes,slices,nodes_per_slice,fast_mode=None):
+def define_test (keys,sites,persons,nodes,slices,
+                 nodes_per_slice,persons_per_slice,fast_mode=None):
     global number_keys, number_sites, number_persons, number_nodes, number_slices
-    global number_nodes_per_slice, fast_flag
+    global number_nodes_per_slice, number_persons_per_slice, fast_flag
     number_keys=keys
     number_sites = sites
     number_persons=persons
     number_nodes=nodes
     number_slices=slices
     number_nodes_per_slice=nodes_per_slice
+    number_persons_per_slice=persons_per_slice
     if fast_mode is not None:
         fast_flag=fast_mode
 
@@ -99,10 +102,10 @@ def show_test():
         print 'Running locally on index %d'%local_index
 
 def mini():
-    define_test(1,1,1,1,1,1,True)
+    define_test(1,1,1,1,1,1,1,True)
     
 def normal():
-    define_test (keys=2,sites=4,persons=4,nodes=5,slices=4,nodes_per_slice=5,fast_mode=False)
+    define_test (keys=2,sites=4,persons=4,nodes=5,slices=4,nodes_per_slice=5,persons_per_slice=8,fast_mode=False)
 
 # use only 1 key in this case
 big_factor=4
@@ -563,7 +566,7 @@ def test03_node_n (nn,args=[1,2]):
         except:
            login_base=site_login_base(i,map_on_site(nn))
             n=s[i].AddNode(a[i],login_base,{'hostname': nodename})
-            print '%02d:== Added node %d %s'%(i,n,node_name(i,i))
+            print '%02d:== Added node %d %s'%(i,n,node_name(i,nn))
 
 def test02_delnode (args=[1,2]):
     for nn in myrange(number_nodes):
@@ -606,7 +609,8 @@ def test04_slice_n (ns,args=[1,2]):
                                           'instanciation':'plc-instantiated',
                                           })
             print '%02d:== created slice %d - max nodes=%d'%(i,slice_id,max_nodes)
-           for np in myrange(number_persons):
+            actual_persons_per_slice = min (number_persons,number_persons_per_slice)
+           for np in myrange(actual_persons_per_slice):
                email = person_name (i,np)
                retcod = s[i].AddPersonToSlice (a[i], email, slicename)
                print '%02d:== Attached person %s to slice %s'%(i,email,slicename)
@@ -983,6 +987,7 @@ def test_all ():
     message("END")
 
 ### ad hoc test sequences
+# we just create objects here so we can dump the DB
 def populate ():
     timer_start()
     test_all_init()
@@ -1001,13 +1006,17 @@ def populate ():
     timer_show()
     test05_sa([1])
     timer_show()
-#    test00_refresh ("populate: refreshing peer 1",[1])
-#    timer_show()
-#    test04_slice_add_fnode([1])
-#    timer_show()
-#    test00_refresh("populate: refresh all")
+
+def populate_end():
+    test00_init()
+    test00_refresh ("populate: refreshing peer 1",[1])
+    timer_show()
+    test04_slice_add_fnode([1])
+    timer_show()
+    test00_refresh("populate: refresh all")
     dump()
     timer_show()
+    message("END")
 
 # temporary - scratch as needed
 def test_now ():
@@ -1022,6 +1031,7 @@ def usage ():
     print "Usage: %s [-n] [-f]"%sys.argv[0]
     print " -n runs test_now instead of test_all"
     print " -p runs populate instead of test_all"
+    print " -e runs populate_end of test_all"
     print " -m run in mini mode (1 instance of each class)"
     print " -b performs big run (%d times as large as normal)"%big_factor
     print " -H performs huge run (%d times as large as normal)"%huge_factor
@@ -1031,7 +1041,7 @@ def usage ():
 
 def main ():
     try:
-        (o,a) = getopt.getopt(sys.argv[1:], "mnpbHl:")
+        (o,a) = getopt.getopt(sys.argv[1:], "emnpbHl:")
     except:
         usage()
     func = test_all
@@ -1042,6 +1052,9 @@ def main ():
         elif opt=='-p':
            print 'Running populate'
             func = populate
+        elif opt=='-e':
+           print 'Running populate_end'
+            func = populate_end
         elif opt=='-m':
             mini()
        elif opt=='-b':

diff --git a/peers-test.mk b/peers-test.mk
index e9b508c..a2cbaaa 100644 (file)
--- a/peers-test.mk
+++ b/peers-test.mk
@@ -8,6 +8,8 @@ APIDIR=/usr/share/plc_api
 
 PLC1=lurch.cs.princeton.edu
 PLC2=planetlab-devbox.inria.fr
+PLC1SSH=root@$(PLC1)
+PLC2SSH=root@$(PLC2)
 
 all:help
 
@@ -31,14 +33,20 @@ pplc2:
        rsync -a -v -C ./planetlab4.sql ./PLC root@$(PLC2):$(CHROOT)$(APIDIR)/
 
 ####################
-DB=install-schema stop-clients clean-db restart-db
+DB=install-schema stop-clients db-drop restart-full-db
+DBI= stop-clients db-drop restart-db db-restore restart-http
 WEB=install-api restart
 
 db: $(DB)
        @date
 
+dbi: $(DBI)
+       @echo Restored $(DBDUMP) on $(shell hostname) at $(shell date)
+
+DBDUMP=planetlab4.dump
+
 db-dump:
-       chroot $(CHROOT) pg_dump -U pgsqluser planetlab4 > planetlab4.dump
+       chroot $(CHROOT) pg_dump -c -U pgsqluser planetlab4 > $(DBDUMP)
 
 web: $(WEB)
 
@@ -56,27 +64,39 @@ stop-clients:
        @echo 'pkilling Shell.py'
        @-pkill Shell.py
        @echo stopping httpd
-       @chroot $(CHROOT) /etc/plc.d/httpd stop
+       service plc stop httpd
+
+db-drop:
+       echo 'dropping db'
+       chroot $(CHROOT) psql -U postgres --port $(PORT) template1 -c 'drop database planetlab4'
 
-clean-db:
-       @echo 'dropping db'
-       @chroot $(CHROOT) psql -U postgres --port $(PORT) template1 -c 'drop database planetlab4'
+db-restore: 
+       echo Restoring $(DBDUMP)
+       rm -f $(DBDUMP).rest-log  $(DBDUMP).rest-err
+       chroot $(CHROOT) psql -U postgres --port $(PORT) -d planetlab4 < $(DBDUMP) > $(DBDUMP).rest-log 2> $(DBDUMP).rest-err
+       ls -l $(DBDUMP).rest-log  $(DBDUMP).rest-err
 
 restart-db:
-       @echo 'restarting db'
-       @chroot $(CHROOT) service plc stop db postgresql httpd
-       @chroot $(CHROOT) service plc start httpd postgresql db
+       @echo 'restarting db only'
+       service plc stop postgresql
+       service plc start postgresql
+
+restart-full-db:
+       @echo 'restarting full db'
+       service plc stop db postgresql httpd
+       service plc start httpd postgresql db
 
 restart:
        @echo 'Restarting PLC'
        @chroot $(CHROOT) service plc restart
 
-http:
+restart-http:
        @echo 'Restarting httpd'
-       @chroot $(CHROOT) /etc/plc.d/httpd stop ; chroot $(CHROOT) /etc/plc.d/httpd start
+       service plc stop httpd
+       service plc start httpd
 
 ####################
-UPGRADE=down up reconfig restart
+UPGRADE=stop-clients down clean-plc up reconfig restart
 
 upgrade: $(UPGRADE)
 
@@ -86,6 +106,8 @@ rpm:
 down:
        cp $(SITEDIR)/$(SITE) .
        rpm -e myplc
+clean-plc:
+       rm -rf /plc
 up:
        rpm -i $(RPM)
 
@@ -111,22 +133,9 @@ log:
 normalize      = egrep -v "'expires':|^+++.*ellapsed"
 
 TEST=run checkpoint diff
-run: nrun normalize 
+run: run-n normalize 
 normalize: TestPeers-n.nout TestPeers-n.nref
 
-nrun: 
-       python -u ./TestPeers.py > TestPeers-n.out 2>&1
-mrun:
-       python -u ./TestPeers.py -m > TestPeers-m.out 2>&1
-brun:
-       python -u ./TestPeers.py -b > TestPeers-b.out 2>&1
-prun:
-       python -u ./TestPeers.py -p > TestPeers-p.out 2>&1
-pbrun:
-       python -u ./TestPeers.py -p -b > TestPeers-pb.out 2>&1
-phrun:
-       python -u ./TestPeers.py -p -H > TestPeers-ph.phout 2>&1
-
 %.nout: %.out
        $(normalize) $*.out > $@
 %.nref: %.ref
@@ -141,22 +150,126 @@ ckp checkpoint:
        cp TestPeers-n.out TestPeers-n.ref
        rm -f TestPeers-n.n???
 
-mdiff: TestPeers-m.nref TestPeers-m.nout 
+# variant runs
+VARIANT-TESTS :=
+
+VARIANT-TESTS += run-n
+run-n: 
+       python -u ./TestPeers.py > TestPeers-n.out 2>&1
+VARIANT-TESTS += run-m
+run-m:
+       python -u ./TestPeers.py -m > TestPeers-m.out 2>&1
+VARIANT-TESTS += run-b
+run-b:
+       python -u ./TestPeers.py -b > TestPeers-b.out 2>&1
+VARIANT-TESTS += run-p run-pn
+run-p run-pn:
+       python -u ./TestPeers.py -p > TestPeers-p.out 2>&1
+VARIANT-TESTS += run-pb
+run-pb:
+       python -u ./TestPeers.py -p -b > TestPeers-pb.out 2>&1
+VARIANT-TESTS += run-ph
+run-ph:
+       python -u ./TestPeers.py -p -H > TestPeers-ph.out 2>&1
+VARIANT-TESTS += run-e run-en
+run-e run-en:
+       python -u ./TestPeers.py -e > TestPeers-e.out 2>&1
+VARIANT-TESTS += run-eb
+run-eb:
+       python -u ./TestPeers.py -e -b > TestPeers-eb.out 2>&1
+VARIANT-TESTS += run-eh
+run-eh:
+       python -u ./TestPeers.py -e -H > TestPeers-eh.out 2>&1
+
+VARIANT-TESTS += diff-m
+diff-m: TestPeers-m.nref TestPeers-m.nout 
        diff TestPeers-m.nref TestPeers-m.nout
-mckp:
+VARIANT-TESTS += ckp-m
+ckp-m:
        cp TestPeers-m.out TestPeers-m.ref
        rm -f TestPeers-m.n???
 
+VARIANT-TESTS += diff-p
+diff-p: TestPeers-p.nref TestPeers-p.nout 
+       diff TestPeers-p.nref TestPeers-p.nout
+VARIANT-TESTS += ckp-p
+ckp-p:
+       cp TestPeers-p.out TestPeers-p.ref
+       rm -f TestPeers-p.n???
+
+
+VARIANTS-DB := 
+
+DB1=populate-1.sql
+DB2=populate-2.sql
+
+SAVE=save1 save2
+VARIANT-DB += save
+save: $(SAVE)
+
+VARIANT-DB += save-n
+save-n: DB1=populate-n-1.sql
+save-n: DB2=populate-n-2.sql
+save-n:save
+
+VARIANT-DB += save-b
+save-b: DB1=populate-b-1.sql
+save-b: DB2=populate-b-2.sql
+save-b:save
+
+VARIANT-DB += save-h
+save-h: DB1=populate-h-1.sql
+save-h: DB2=populate-h-2.sql
+save-h:save
+
+save1:
+       ssh $(PLC1SSH) "make -C new_plc_api -f peers-test.mk DBDUMP=$(DB1) db-dump"
+       scp $(PLC1SSH):new_plc_api/$(DB1) .
+save2:
+       ssh $(PLC2SSH) "make -C new_plc_api -f peers-test.mk DBDUMP=$(DB2) db-dump"
+       scp $(PLC2SSH):new_plc_api/$(DB2) .
+
+RESTORE=restore1 restore2
+VARIANT-DB += restore
+restore:$(RESTORE)
+
+VARIANT-DB += restore-n
+restore-n: DB1=populate-n-1.sql
+restore-n: DB2=populate-n-2.sql
+restore-n:restore
+
+VARIANT-DB += restore-b
+restore-b: DB1=populate-b-1.sql
+restore-b: DB2=populate-b-2.sql
+restore-b:restore
+
+VARIANT-DB += restore-h
+restore-h: DB1=populate-h-1.sql
+restore-h: DB2=populate-h-2.sql
+restore-h:restore
+
+restore1:
+       scp $(DB1) $(PLC1SSH):new_plc_api/
+       ssh $(PLC1SSH) "make -C  new_plc_api -f peers-test.mk DBDUMP=$(DB1) dbi"
+restore2:
+       scp $(DB2) $(PLC2SSH):new_plc_api/
+       ssh $(PLC2SSH) "make -C  new_plc_api -f peers-test.mk DBDUMP=$(DB2) dbi"
+
 #######
-HELP=rpm db-dump http
+HELP=rpm db-dump restart-http
 
 help:
        @echo known targets:
        @echo push: $(PUSH) 
        @echo db: $(DB) 
+       @echo dbi: $(DBI) 
+       @echo run: $(RUN)
        @echo web: $(WEB) 
        @echo upgrade: $(UPGRADE)
        @echo test: $(TEST)
-       @echo run: $(RUN)
+       @echo other test targets: $(VARIANT-TESTS)
+       @echo save:$(SAVE)
+       @echo restore:$(RESTORE)
+       @echo db targets: $(VARIANT-DB)
        @echo OTHERS: $(HELP)