small modification to th experiment runner

[nepi.git] / src / nepi / execution / runner.py

diff --git a/src/nepi/execution/runner.py b/src/nepi/execution/runner.py
index 52b4bde..2a78be0 100644 (file)
--- a/src/nepi/execution/runner.py
+++ b/src/nepi/execution/runner.py
@@ -22,7 +22,6 @@ from nepi.execution.ec import ExperimentController, ECState
 import math
 import numpy
 import os
-import tempfile
 import time
 
 class ExperimentRunner(object):
@@ -59,7 +58,7 @@ class ExperimentRunner(object):
         :param compute_metric_callback: function to invoke after each 
             experiment run, to compute an experiment metric. 
             It will be invoked with the ec and the run count as arguments,
-            and it must return a numeric value for the computed metric:
+            and it must return the metric value(s) computed for the run
 
                 metric = compute_metric_callback(ec, run)
             
@@ -69,7 +68,7 @@ class ExperimentRunner(object):
             collected metric samples have converged and the experiment runner
             can stop. It will be invoked with the ec, the run count and the
             list of collected metric samples as argument, and it must return
-            either True or False:
+            either True or False
 
                 stop = evaluate_convergence_callback(ec, run, metrics)
 
@@ -92,21 +91,19 @@ class ExperimentRunner(object):
         # Force persistence of experiment controller
         ec._persist = True
 
-        dirpath = tempfile.mkdtemp()
-        filepath = ec.save(dirpath)
+        filepath = ec.save(dirpath = ec.exp_dir)
 
         samples = []
         run = 0
-        while True: 
+        stop = False
+
+        while not stop: 
             run += 1
 
             ec = self.run_experiment(filepath, wait_time, wait_guids)
             
             ec.logger.info(" RUN %d \n" % run)
 
-            if run >= min_runs and max_runs > -1 and run >= max_runs :
-                break
-
             if compute_metric_callback:
                 metric = compute_metric_callback(ec, run)
                 if metric is not None:
@@ -114,7 +111,11 @@ class ExperimentRunner(object):
 
                     if run >= min_runs and evaluate_convergence_callback:
                         if evaluate_convergence_callback(ec, run, samples):
-                            break
+                            stop = True
+
+            if run >= min_runs and max_runs > -1 and run >= max_runs :
+                stop = True
+
             del ec
 
         return run
@@ -123,18 +124,21 @@ class ExperimentRunner(object):
         if len(samples) == 0:
             msg = "0 samples collected"
             raise RuntimeError, msg
-        
+
         x = numpy.array(samples)
         n = len(samples)
         std = x.std()
         se = std / math.sqrt(n)
         m = x.mean()
-        se95 = se * 2
+
+        # confidence interval for 95% confidence level.
+        # Asuming samples are normally distributed
+        ci95 = se * 2
         
-        ec.logger.info(" RUN %d - SAMPLES %d MEAN %.2f STD %.2f SE95%% %.2f \n" % (
-            run, n, m, std, se95 ) )
+        ec.logger.info(" RUN %d - SAMPLES %d MEAN %.2f STD %.2f CI (95%%) %.2f \n" % (
+            run, n, m, std, ci95 ) )
 
-        return m * 0.05 >= se95
+        return m * 0.05 >= ci95
 
     def run_experiment(self, filepath, wait_time, wait_guids): 
         ec = ExperimentController.load(filepath)
@@ -151,4 +155,3 @@ class ExperimentRunner(object):
 
         return ec
 
-