test/nightly rewrite

2 files changed, 80 insertions, 522 deletions

diff --git a/nightly/nightly.rb b/nightly/nightly.rb
index dc06ae3..594d4ee 100644
--- a/nightly/nightly.rb
+++ b/nightly/nightly.rb
@@ -1,4 +1,5 @@
 
+require 'test/test_examples.rb'
 
 class Nightly
   
@@ -15,18 +16,15 @@ class Nightly
     end
   end
   
-  def self.build_nightly(dry_run=false)
+  def self.build_nightly(select=nil, dry_run=false)
+    
+    validationExamples = ValidationExamples.select(select)
+    return "please \"select\" validation examples:\n"+ValidationExamples.list if validationExamples.size==0
+    
     task_uri = OpenTox::Task.as_task() do
       LOGGER.info("Building nightly report")
       
-      benchmarks = [ HamsterTrainingTestBenchmark.new,
-                     #HamsterCrossvalidationBenchmark.new, 
-                     #MiniRegressionBenchmark.new,
-                     #CacoModelsRegressionBenchmark.new,
-                     #CacoModelsRegressionBenchmark2.new,
-                     #CacoAlgsRegressionBenchmark.new,
-                     #FatheadRegressionBenchmark.new,
-                     ]
+      benchmarks = validationExamples.collect{ |e| ValidationBenchmark.new(e) }
       
       running = []
       report = Reports::XMLReport.new("Nightly Validation", Time.now.strftime("Created at %m.%d.%Y - %H:%M"))
@@ -60,15 +58,14 @@ class Nightly
          section = report.add_section(report.get_root_element, b.title)
          
          section_info = report.add_section(section, "Info")
-         b.info.each{|i| report.add_paragraph(section_info,i)}
          info_table = b.info_table
-         report.add_table(section_info, b.info_table_title, info_table) if info_table
+         report.add_table(section_info, "Validation info", info_table) if info_table
          
          section_results = report.add_section(section, "Results")
-         report.add_table(section_results, b.result_table_title, b.result_table)
+         report.add_table(section_results, "Valdation results", b.result_table)
          
          if (b.comparison_report)
-           report.add_table(section_results, b.comparison_report_title, [[b.comparison_report]], false)
+           report.add_table(section_results, "Validation comparison report", [[b.comparison_report]], false)
          end
          
          section_errors = report.add_section(section, "Errors")
@@ -96,7 +93,7 @@ class Nightly
       else
         LOGGER.info("Nightly report completed - DRY RUN, no report creation")
       end
-      return "Nightly report completed"
+      benchmarks.collect{|b| b.uris}.join(",")
     end
     if defined?(halt)
       halt 202,task_uri
@@ -105,120 +102,102 @@ class Nightly
     end
   end
   
-  class AbstractBenchmark
-    
-    def info_table_title
-      return title
+  class ValidationBenchmark
+  
+    attr_accessor :errors, :comparison_report
+
+    def comparable
+      if @comp == nil
+        @comp = @validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri 
+      end
+      @comp
     end
     
-    def info_table
-      return nil
+    def uris
+      @validation_examples.collect{|v| v.validation_uri}.join(",")
+    end
+
+    def initialize(validationExamples)
+      @validation_examples = []
+      validationExamples.each do |v|
+        example = v.new
+        @validation_examples << example
+      end
     end
     
-    def result_table_title
-      return "Validation results"
+    def title
+      @validation_examples.collect{|e| e.title}.join(" + ")
     end
     
     def result_table
-      raise "no comparables" unless @comparables
-      raise "no validations" unless @validations
-      raise "no reports" unless @reports
       t = []
-      row = [comparable_nice_name, "validation", "report"]
+      row = [comparable.to_s, "validation", "report"]
       t << row
-      (0..@comparables.size-1).each do |i|
-        row = [ @comparables[i], @validations[i], @reports[i] ]
+      @validation_examples.each do |e|
+        row = [ e.send(comparable), 
+                (e.validation_error!=nil ? "error, see below" : e.validation_uri),
+                (e.report_error!=nil ? "error, see below" : e.report_uri) ]
         t << row
       end
       t
     end
-    
-    def comparison_report_title
-      "algorithm comparison report"
-    end
-    
-    def comparison_report
-      @comparison_report
-    end
-    
-  end
-  
-  class ValidationBenchmark < AbstractBenchmark
 
     def info_table
-      raise "no comparables" unless @comparables
       t = []
       t << ["param", "uri"]
-      params.each do |k,v|
-        t << [k.to_s, v.to_s]
-      end
-      count = 1
-      @comparables.each do |alg|
-        t << [comparable_nice_name+" ["+count.to_s+"]", alg]
-        count += 1
+      
+      (@validation_examples[0].params+@validation_examples[0].opt_params).each do |p|
+        map = {}
+        @validation_examples.each{|e| map[e.send(p).to_s]=nil }
+        
+        if map.size==1 && map.keys[0].size==0
+          #omit
+        elsif map.size==1 #values equal
+          t << [p.to_s, map.keys[0]]
+        else
+          count = 1
+          @validation_examples.each do |e|
+            t << [p.to_s+" ["+count.to_s+"]", e.send(p)]
+            count += 1
+          end
+        end
       end
       t
     end
     
-    def errors
-      @errors
-    end
-    
-    def params
-      raise "return uri-value hash"
-    end
-    
-    def validate(index)
-      raise "validate, return uri"
-    end
-    
-    def build_report(index)
-      raise "build report, return uri"
-    end
-    
-    def build_compare_report(comparables)
-      raise "build compare report, return uri"
-    end
-    
     def build()
-      raise "no comparables" unless @comparables
       
-      @validations = Array.new(@comparables.size)
-      @reports = Array.new(@comparables.size)
       @errors = {}
-      to_compare = []
-#      LOGGER.info "train-data: "+@train_data.to_s
-#      LOGGER.info "test-data: "+@test_data.to_s
-#      LOGGER.info "test-class-data: "+@test_class_data.to_s
       
       running = []
-      (0..@comparables.size-1).each do |i|
-        
+      count = 1
+      
+      @validation_examples.each do |v|
+
+        id = v.title+count.to_s
+        count += 1
+        running << id
+        LOGGER.debug "Uploading datasets: "+v.title
+        v.upload_files
+        v.check_requirements
+
         Thread.new do 
-          running << @comparables[i]+i.to_s
-          begin
-            LOGGER.debug "Validate: "+@comparables[i].to_s
-            @validations[i] = validate(i)
-            to_compare << @validations[i] if OpenTox::Utils.is_uri?(@validations[i])
-              
-            begin
-              LOGGER.debug "Building validation-report for "+@validations[i].to_s+" ("+@comparables[i].to_s+")"
-              @reports[i] = build_report(i)
-            rescue => ex
-              LOGGER.error "validation-report error: "+ex.message
-              @reports[i] = "error"
+          
+          LOGGER.debug "Validate: "+v.title
+          v.validate
+          if v.validation_error!=nil
+            @errors["Error validating "+v.title] = v.validation_error
+          else
+            LOGGER.debug "Building report: "+v.title
+            v.report
+            if v.report_error!=nil
+              @errors["Error building report for "+v.title] = v.report_error
             end
-            
-          rescue => ex
-            LOGGER.error "validation error: "+ex.message
-            key = "Error validating "+@comparables[i].to_s
-            @validations[i] = key+" (see below)"
-            @errors[key] = ex.message
-          ensure
-            running.delete(@comparables[i]+i.to_s)
           end
+          running.delete(id)
         end
       end
+
       wait = 0
       while running.size>0
         LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+" waiting for "+running.inspect if wait%20==0
@@ -226,430 +205,9 @@ class Nightly
         sleep 1
       end
       
-      if to_compare.size>1
-        LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": build comparison report"
-        @comparison_report = build_compare_report(to_compare)
-      else
-        LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": nothing to compare"
-      end
-    end
-  end
-  
-  class AlgorithmValidationBenchmark < ValidationBenchmark
-
-    def comparable_nice_name
-      return "algorithm"
-    end
-    
-    def build()
-      raise "no algs" unless @algs
-      @comparables = @algs
-      super
-    end  
-    
-    def build_compare_report(comparables)
-      Util.create_alg_comparison_report(comparables)
-    end
-  end
-  
-  class ModelValidationBenchmark < ValidationBenchmark
-
-    def comparable_nice_name
-      return "model"
-    end
-    
-    def build()
-      raise "no models" unless @models
-      @comparables = @models
-      super
-    end
-    
-    def build_compare_report(comparables)
-      "model comparsion report not available yet" #Util.create_model_comparison_report(comparables)
-    end
-  end
-  
-  class TestModelValidationBenchmark < ModelValidationBenchmark
-    
-    def info
-      [ training_test_info ]
-    end
-    
-    def training_test_info
-      "This is a test set validation of existing models. "+
-      "The model is used to predict the test dataset. Evaluation is done by comparing the model predictions "+
-      "to the actual test values (in the test (target) dataset)."
-    end
-    
-    def info_table_title
-      "Validation params"
-    end
-    
-    def params
-      p = { "test_dataset_uri" => @test_data }
-      p["test_target_dataset_uri"] = @test_class_data if @test_class_data
-      return p
-    end
-    
-    def validate(index)
-      Util.validate_model(@test_data, @test_class_data, @models[index])
-    end
-    
-    def build_report(index)
-      Util.create_report(@validations[index])
-    end
-      
-    def build()
-      raise "no test data" unless @test_data
-      super
-    end
-  end
-  
-  class TrainingTestValidationBenchmark < AlgorithmValidationBenchmark
-    
-    def info
-      [ training_test_info ]
-    end
-    
-    def training_test_info
-      "This is a training test set validation. It builds a model with an algorithm and the training dataset. "+
-      "The model is used to predict the test dataset. Evaluation is done by comparing the model predictions "+
-      "to the actual test values (in the test target dataset)."
-    end
-    
-    def info_table_title
-      "Validation params"
-    end
-    
-    def params
-      p = { "training_dataset_uri" => @train_data, "test_dataset_uri" => @test_data,
-                "prediction_feature" => @pred_feature }
-      p["test_target_dataset_uri"] = @test_class_data if @test_class_data
-      return p
-    end
-    
-    def validate(index)
-      Util.validate_alg(@train_data, @test_data, @test_class_data,
-              @algs[index], @pred_feature, @alg_params[index])
-    end
-    
-    def build_report(index)
-      Util.create_report(@validations[index])
-    end
-      
-    def build()
-      raise "no train data" unless @train_data
-      raise "no test data" unless @test_data
-      raise "no pred feature" unless @pred_feature
-      super
-    end
-  end
-  
-  class CrossValidationBenchmark < AlgorithmValidationBenchmark
-    
-    def info
-      [ training_test_info ]
-    end
-    
-    def training_test_info
-      "This is a cross-validation."
-    end
-    
-    def info_table_title
-      "Cross-validation params"
+      LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": build comparison report"
+      @comparison_report = ValidationExamples::Util.build_compare_report(@validation_examples)
     end
-    
-    def params
-      p = { "dataset_uri" => @data, "prediction_feature" => @pred_feature,
-            "num_folds" => @num_folds, "random_seed" => @random_seed, "stratified" => @stratified}
-      return p
-    end
-    
-    def validate(index)
-      Util.cross_validate_alg(@data, @algs[index], @pred_feature, 
-              @num_folds, @random_seed, @stratified, @alg_params[index])
-    end
-    
-    def build_report(index)
-      Util.create_report(@validations[index], "crossvalidation")
-    end
-    
-    def build()
-      raise "no data" unless @data
-      raise "no pred feature" unless @pred_feature
-      @num_folds = 10 unless @num_folds
-      @random_seed = 1 unless @random_seed
-      @stratified = false unless @stratified
-      super
-    end
-  end
-  
-  class HamsterCrossvalidationBenchmark < CrossValidationBenchmark
-    
-    @@dataset_service = @@config[:services]["opentox-dataset"]
-    @@file=File.new("data/hamster_carcinogenicity.yaml","r")
-    @@file_type="text/x-yaml"
-    @@lazar_server = @@config[:services]["opentox-algorithm"]
-    
-    def title()
-      "Classification, Crossvalidation, Hamster carcinogenicity"
-    end
-    
-    def info
-      res = [ "A crossvalidation using the hamster carcinogenicity dataset." ] + super
-      return res
-    end
-    
-    def build()
-      @algs = [
-        #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]),
-        #File.join(@@lazar_server,"lazar"),
-        "http://ot-test.in-silico.ch/algorithm/lazar",
-        #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]),
-        #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]),
-        ]
-      @alg_params = [
-        #nil,
-        #"feature_generation_uri="+File.join(@@lazar_server,"fminer"),
-        "feature_generation_uri=http://ot-test.in-silico.ch/algorithm/fminer",
-        nil,
-        nil
-        ]
-      #@pred_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
-      @pred_feature = "http://ot-test.in-silico.ch/toxcreate/feature#Hamster%20Carcinogenicity%20(CPDB/DSSTOX)"
-      @data = "http://ot-test.in-silico.ch/dataset/1"
-      #LOGGER.debug "upload hamster datasets"
-      #@data = Util.upload_dataset(@@dataset_service, @@file, @@file_type).chomp("\n")
-      super
-    end
-  end
-  
-  class HamsterTrainingTestBenchmark < TrainingTestValidationBenchmark
-    
-    @@dataset_service = @@config[:services]["opentox-dataset"]
-    @@file=File.new("data/hamster_carcinogenicity.yaml","r")
-    @@file_type="text/x-yaml"
-    @@lazar_server = @@config[:services]["opentox-algorithm"]
-    
-    def title()
-      "Classification, Algorithm training test validation, Hamster carcinogenicity"
-    end
-    
-    def info
-      res = [ "A simple binary classification task using the hamster carcinogenicity dataset." ] + super
-      return res
-    end
-    
-    def build()
-      @algs = [
-        #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]),
-        File.join(@@lazar_server,"lazar"),
-        #"http://188.40.32.88/algorithm/lazar",
-        ]
-      @alg_params = [
-        #nil,
-        "feature_generation_uri="+File.join(@@lazar_server,"fminer"),
-        #"feature_generation_uri=http://188.40.32.88/algorithm/fminer",
-        ]
-      
-      LOGGER.debug "prepare hamster datasets"
-      
-      @pred_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
-      @test_class_data = Util.upload_dataset(@@dataset_service, @@file, @@file_type).chomp("\n")
-      #@pred_feature = "http://188.40.32.88/toxcreate/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
-      #@test_class_data = "http://188.40.32.88/dataset/57"
-      
-      split = Util.split_dataset(@test_class_data, @pred_feature, 0.9, 1)
-      @train_data = split[0].to_s
-      @test_data = split[1].to_s
-      raise "could not split "+@train_data.to_s+" "+@test_data.to_s unless OpenTox::Utils.is_uri?(@train_data) and OpenTox::Utils.is_uri?(@test_data) 
-      super
-    end
-  end
-  
-  
-  class MiniRegressionBenchmark < TrainingTestValidationBenchmark
-    
-    def title
-      "Regression, Algorithm training test validation, tiny test dataset"
-    end
-    
-    def info
-      res = [ "A very small regression task, using the training dataset as test set." ] + super
-      return res
-    end
-    
-    def build()
-      @algs = [ 
-        "http://opentox.ntua.gr:3003/algorithm/mlr",
-        "http://opentox.ntua.gr:3004/algorithm/mlr",
-        "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNregression",
-        File.join(@@config[:services]["opentox-majority"],["/regr/algorithm"])
-        ]
-      @alg_params = [ nil, "dataset_service=http://ambit.uni-plovdiv.bg:8080/ambit2/dataset", nil]
-      @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
-      @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
-      @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
-      super
-    end
-  end
-  
-  class CacoAlgsRegressionBenchmark < TrainingTestValidationBenchmark
-    
-    def title
-      "Regression, Algorithm training test validation, Caco-2 dataset"
-    end
-    
-    def info
-      res = [ "Training test set validation on caco2 dataset." ] + super
-      return res
-    end
-    
-    def build()
-      @algs = [ 
-        "http://opentox.ntua.gr:3003/algorithm/mlr",
-        "http://opentox.ntua.gr:3004/algorithm/mlr",
-        "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/LR",
-        ]
-      @alg_params = [ nil, nil]
-      @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798"
-      @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353"
-      @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510"
-      super
-    end
-  end
-
-  
-  class CacoModelsRegressionBenchmark < TestModelValidationBenchmark
-    
-    def title
-      "Regression, Model test set validation, Caco-2 dataset (ambit.uni-plovdiv)"
-    end
-    
-    def info
-      res = [ "Valdation of two identical(?) mlr models on caco-2 dataset." ] + super
-      return res
-    end
-    
-    def build()
-      @models = [ 
-        "http://ambit.uni-plovdiv.bg:8080/ambit2/model/259260",
-        "http://opentox.ntua.gr:3003/model/195",
-        ]
-      #@test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798"
-      @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353"
-      super
-    end
-  end
-  
-    
-  class CacoModelsRegressionBenchmark2 < TestModelValidationBenchmark
-    
-    def title
-      "Regression, Model test set validation, Caco-2 dataset (apps.ideaconsult)"
-    end
-    
-    def info
-      res = [ "One more caco2 dataset at another location" ] + super
-      return res
-    end
-    
-    def build()
-      @models = [ 
-        "http://apps.ideaconsult.net:8080/ambit2/model/33",
-        ]
-      @test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
-      #prediction_feature=http://apps.ideaconsult.net:8080/ambit2/feature/22200
-      super
-    end
-  end
-  
-  
-  
-  class FatheadRegressionBenchmark < TrainingTestValidationBenchmark
-    
-    def title
-      "Training test set validation, regression with fathead minnow dataset"
-    end
-    
-    def info
-      res = [ "This is the regression use case used in D2.2. "+
-              "The task is to predict LC50 values of the well known Fathead Minnow Acute Toxicity dataset. "+
-              "JOELIB was used to compute numerical descriptors as features." ] + super
-      return res
-    end
-    
-    def build()
-      @algs = [ 
-        "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNregression",
-        "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/M5P",
-        "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/GaussP"
-        ]
-      @alg_params = [nil, nil, nil]
-      @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/639"
-      @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/640"
-      @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/264185"
-      super
-    end
-  end
- 
-  
-
-
-  class Util
-    @@validation_service = @@config[:services]["opentox-validation"]
-    
-    def self.upload_dataset(dataset_service, file, file_type)
-      raise "File not found: "+file.path.to_s unless File.exist?(file.path)
-      data = File.read(file.path)
-      data_uri = OpenTox::RestClientWrapper.post dataset_service, {:content_type => file_type}, data
-      #data_uri = OpenTox::Task.find(data_uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(data_uri)
-      return data_uri.to_s
-    end
-    
-    def self.split_dataset(data_uri, feature, split_ratio, random_seed)
-      res = OpenTox::RestClientWrapper.post File.join(@@validation_service,'plain_training_test_split'), { :dataset_uri => data_uri, :prediction_feature=>feature, :split_ratio=>split_ratio, :random_seed=>random_seed}
-      return res.split("\n")
-    end
-    
-    def self.validate_alg(train_data, test_data, test_class_data, alg, feature, alg_params)
-      uri = OpenTox::RestClientWrapper.post @@validation_service, { :training_dataset_uri => train_data, :test_dataset_uri => test_data, 
-        :test_target_dataset_uri => test_class_data,
-        :algorithm_uri => alg, :prediction_feature => feature, :algorithm_params => alg_params }
-      #LOGGER.info "waiting for validation "+uri.to_s
-      #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
-      #LOGGER.info "validaiton done "+uri.to_s
-      return uri.to_s
-    end
-    
-    def self.validate_model(test_data, test_class_data, model)
-      uri = OpenTox::RestClientWrapper.post @@validation_service, { :test_dataset_uri => test_data, 
-        :test_target_dataset_uri => test_class_data, :model_uri => model }
-      return uri.to_s
-    end    
-    
-    def self.cross_validate_alg(data, alg, feature, folds, seed, stratified, alg_params)
-      uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"crossvalidation"), { :dataset_uri => data, 
-        :algorithm_uri => alg, :prediction_feature => feature, :algorithm_params => alg_params, :num_folds => folds, 
-        :random_seed => seed, :stratified => stratified }
-      #LOGGER.info "waiting for validation "+uri.to_s
-      #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
-      #LOGGER.info "validaiton done "+uri.to_s
-      return uri.to_s
-    end
-    
-    def self.create_report(validation, type="validation")
-      uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"report/"+type.to_s), { :validation_uris => validation }
-      #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
-      return uri.to_s
-    end
-    
-    def self.create_alg_comparison_report(validations)
-      uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"report/algorithm_comparison"), { :validation_uris => validations.join("\n") }
-      #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
-      return uri.to_s
-    end
-    
   end
   
 end
\ No newline at end of file
diff --git a/nightly/nightly_application.rb b/nightly/nightly_application.rb
index c366024..69b4614 100644
--- a/nightly/nightly_application.rb
+++ b/nightly/nightly_application.rb
@@ -1,8 +1,8 @@
 require "nightly/nightly.rb"
 
-get '/build_nightly/?' do
+post '/build_nightly/?' do
   dry_run = params[:dry_run]!=nil and params[:dry_run]
-  Nightly.build_nightly(dry_run)
+  Nightly.build_nightly(params["select"],dry_run)
 end
 
 get '/css_style_sheet/?' do