test/nightly rewrite

3 files changed, 425 insertions, 0 deletions

diff --git a/test/test_application.rb b/test/test_application.rb
new file mode 100644
index 0000000..cbc1057
--- /dev/null
+++ b/test/test_application.rb
@@ -0,0 +1,17 @@
+
+post '/test_validation/?' do
+  validationExamples = ValidationExamples.select(params[:select])
+  return "please \"select\" a single validation example:\n"+ValidationExamples.list if validationExamples.size!=1 or validationExamples[0].size!=1
+  OpenTox::Task.as_task do
+    v = validationExamples[0][0]
+    ex = v.new
+    ex.upload_files
+    ex.check_requirements
+    ex.validate
+    raise ex.validation_error if ex.validation_error
+    ex.report if params[:report]
+    raise ex.report_error if ex.report_error
+    ex.validation_uri + (params[:report] ? ","+ex.report_uri : "")
+  end
+end
+
diff --git a/test/test_examples.rb b/test/test_examples.rb
new file mode 100644
index 0000000..d08e358
--- /dev/null
+++ b/test/test_examples.rb
@@ -0,0 +1,214 @@
+
+require 'test/test_examples_util.rb'
+
+class Class
+  def humanize
+    self.to_s.gsub(/.*::/, "").gsub(/([^^A-Z_])([A-Z])/, '\1-\2').gsub(/_/,"-")
+  end
+end
+
+module ValidationExamples
+  
+  ########################################################################################################
+  
+  class ABSTRACTHamsterSplit < SplitTestValidation
+    def initialize
+      @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
+      @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+    end
+  end
+  
+  class LazarHamsterSplit < ABSTRACTHamsterSplit
+    def initialize
+      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      super
+    end
+  end
+  
+  class MajorityHamsterSplit < ABSTRACTHamsterSplit
+    def initialize
+      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      super
+    end
+  end
+  
+  ########################################################################################################
+
+  class ABSTRACTHamsterTrainingTest < TrainingTestValidation
+    def initialize
+      @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
+      @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r")
+      @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r")
+      @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+    end
+  end
+  
+  class MajorityHamsterTrainingTest < ABSTRACTHamsterTrainingTest
+    def initialize
+      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      super
+    end
+  end
+  
+  class LazarHamsterTrainingTest < ABSTRACTHamsterTrainingTest
+    def initialize
+      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      super
+    end
+  end
+  
+  ########################################################################################################  
+
+  class ABSTRACTHamsterCrossvalidation < CrossValidation
+    def initialize
+      @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
+      @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+    end
+  end
+  
+  class MajorityHamsterCrossvalidation < ABSTRACTHamsterCrossvalidation
+    def initialize
+      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      super
+    end
+  end
+
+  class LazarHamsterCrossvalidation < ABSTRACTHamsterCrossvalidation
+    def initialize
+      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      super
+    end
+  end
+  
+  ########################################################################################################  
+
+  class ABSTRACT_ISTHamsterCrossvalidationInsilico < CrossValidation
+    def initialize
+      @dataset_uri = "http://webservices.in-silico.ch/dataset/108"
+      @prediction_feature = "http://toxcreate.org/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+    end
+  end
+  
+  class MajorityISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico
+    def initialize
+      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      super
+    end
+  end
+  
+  class LazarISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico
+    def initialize
+      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      super
+    end
+  end
+  
+  class ISTLazarISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico
+    def initialize
+      @algorithm_uri = "http://webservices.in-silico.ch/algorithm/lazar"
+      @algorithm_params = "feature_generation_uri=http://webservices.in-silico.ch/algorithm/fminer"
+      super
+    end
+  end
+
+  
+  ########################################################################################################
+  
+  class LR_AmbitCacoModel < ModelValidation
+    def initialize
+      @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/33"
+      @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+      #@prediction_feature=http://apps.ideaconsult.net:8080/ambit2/feature/22200
+    end
+  end
+  
+  ########################################################################################################
+
+  class ABSTRACTCacoTrainingTest < TrainingTestValidation
+    def initialize
+      @training_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798"
+      @test_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353"
+      @prediction_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510"
+    end
+  end
+  
+  class LR_AmbitCacoTrainingTest < ABSTRACTCacoTrainingTest
+    def initialize
+      @algorithm_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/LR"
+      super
+    end
+  end
+  
+  class MLR_NTUA_CacoTrainingTest < ABSTRACTCacoTrainingTest
+    def initialize
+      @algorithm_uri = "http://opentox.ntua.gr:3003/algorithm/mlr"
+      super
+    end
+  end
+  
+  class MLR_NTUA2_CacoTrainingTest < ABSTRACTCacoTrainingTest
+    def initialize
+      @algorithm_uri = "http://opentox.ntua.gr:3004/algorithm/mlr"
+      super
+    end
+  end
+  
+  class MajorityCacoTrainingTest < ABSTRACTCacoTrainingTest
+    def initialize
+      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm")
+      super
+    end
+  end
+  
+  ########################################################################################################
+    
+  @@list = {
+      "1" => [ LazarHamsterSplit, MajorityHamsterSplit ],
+      "1a" => [ LazarHamsterSplit ],
+      "1b" => [ MajorityHamsterSplit ],
+      
+      "2" => [ LazarHamsterTrainingTest, MajorityHamsterTrainingTest ],
+      "2a" => [ LazarHamsterTrainingTest ],
+      "2b" => [ MajorityHamsterTrainingTest ],
+      
+      "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation ],
+      "3a" => [ LazarHamsterCrossvalidation ],
+      "3b" => [ MajorityHamsterCrossvalidation ],
+      
+      "4" => [ MajorityISTHamsterCrossvalidation, LazarISTHamsterCrossvalidation, ISTLazarISTHamsterCrossvalidation ],
+      "4a" => [ MajorityISTHamsterCrossvalidation ],
+      "4b" => [ LazarISTHamsterCrossvalidation ],
+      "4c" => [ ISTLazarISTHamsterCrossvalidation ],
+      
+      "5a" => [ LR_AmbitCacoModel ],
+      
+      "6" => [ LR_AmbitCacoTrainingTest, MLR_NTUA_CacoTrainingTest, MLR_NTUA2_CacoTrainingTest, MajorityCacoTrainingTest ],
+      "6a" => [ LR_AmbitCacoTrainingTest ],
+      "6b" => [ MLR_NTUA_CacoTrainingTest ],
+      "6c" => [ MLR_NTUA2_CacoTrainingTest ],
+      "6d" => [ MajorityCacoTrainingTest ],
+    }
+  
+  def self.list
+    @@list.sort.collect{|k,v| k+":\t"+v.collect{|vv| vv.humanize}.join("\n\t")+"\n"} #.join("\n")
+  end
+  
+  def self.select(csv_keys)
+    res = []
+    if csv_keys!=nil and csv_keys.size>0
+      csv_keys.split(",").each do |k|
+        raise "no key "+k.to_s unless @@list.has_key?(k)
+        res << @@list[k]
+      end
+    end
+    return res
+  end
+  
+end
+
+#puts ValidationExamples.list
+#puts ValidationExamples.select("1,2a").inspect
\ No newline at end of file
diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb
new file mode 100644
index 0000000..49e0336
--- /dev/null
+++ b/test/test_examples_util.rb
@@ -0,0 +1,194 @@
+
+module ValidationExamples
+  
+  class Util
+
+    @@dataset_uris = {}
+
+    def self.upload_dataset(file, dataset_service=@@config[:services]["opentox-dataset"], file_type="text/x-yaml")
+      raise "File not found: "+file.path.to_s unless File.exist?(file.path)
+      if @@dataset_uris[file.path.to_s]==nil
+        data = File.read(file.path)
+        data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp
+        @@dataset_uris[file.path.to_s] = data_uri
+        LOGGER.debug "uploaded dataset: "+data_uri
+      else
+        LOGGER.debug "file already uploaded: "+@@dataset_uris[file.path.to_s]
+      end
+      return @@dataset_uris[file.path.to_s]
+    end
+    
+    def self.build_compare_report(validation_examples)
+      
+      @comp = validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri
+      return nil if @comp == :model_uri
+      to_compare = []
+      validation_examples.each do |v|
+        to_compare << v.validation_uri if v.validation_uri and v.validation_error==nil
+      end
+      return nil if to_compare.size < 2
+      begin
+        return validation_post "report/algorithm_comparison",{ :validation_uris => to_compare.join("\n") }
+      rescue => ex
+        return "error creating comparison report "+ex.message
+      end
+    end
+    
+    def self.validation_post(uri, params)
+      if $test_case
+        #puts "posting: "+uri+","+params.inspect
+        $test_case.post uri,params 
+        return wait($test_case.last_response.body)
+      else
+        return OpenTox::RestClientWrapper.post(File.join(@@config[:services]["opentox-validation"],uri),params)
+      end
+    end
+    
+    def self.wait(uri)
+      if OpenTox::Utils.task_uri?(uri)
+        task = OpenTox::Task.find(uri)
+        task.wait_for_completion
+        raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error?
+        uri = task.resultURI
+      end
+      uri
+    end
+    
+  end
+  
+  class ValidationExample
+    
+    #params
+    attr_accessor :name,
+                  :prediction_feature,
+                  :algorithm_uri,
+                  :model_uri,
+                  :test_dataset_uri,
+                  :test_dataset_file,
+                  :test_target_dataset_uri,
+                  :test_target_dataset_file,
+                  :training_dataset_uri,
+                  :training_dataset_file,
+                  :dataset_uri,
+                  :dataset_file,
+                  :algorithm_params,
+                  :split_ratio,
+                  :random_seed,
+                  :num_folds,
+                  :stratified
+    #results                  
+    attr_accessor :validation_uri,
+                  :report_uri,
+                  :validation_error,
+                  :report_error
+    
+    def upload_files
+      [[:test_dataset_uri, :test_dataset_file], 
+       [:test_target_dataset_uri, :test_target_dataset_file],
+       [:training_dataset_uri, :training_dataset_file],
+       [:dataset_uri, :dataset_file]].each do |a|
+         uri = a[0]
+         file = a[1]
+         if send(uri)==nil and send(file)!=nil
+            send("#{uri.to_s}=".to_sym, Util.upload_dataset(send(file)))
+         end
+      end
+    end
+      
+    def check_requirements
+      params.each do |r|
+        raise "values not set: "+r.to_s if send(r)==nil
+      end
+    end
+    
+    def report
+      begin
+        @report_uri = Util.validation_post '/report/'+report_type,{:validation_uris => @validation_uri}
+      rescue => ex
+        @report_error = ex.message
+      end
+    end
+    
+    def validate
+      begin
+        @validation_uri = Util.validation_post '/'+validation_type, get_params
+      rescue => ex
+        @validation_error = ex.message
+      end
+    end
+    
+    def title
+      self.class.humanize
+    end
+    
+    protected
+    def report_type
+      "validation"
+    end
+    
+    def validation_type
+      ""
+    end
+    
+    def get_params
+      p = {}
+      ( params + opt_params ).each do |pp|
+        p[pp] = send(pp) if send(pp)!=nil
+      end
+      return p
+    end
+  end
+  
+  class ModelValidation < ValidationExample
+    
+    def params
+      [:model_uri, :test_dataset_uri]
+    end
+    
+    def opt_params
+      [ :prediction_feature, :test_target_dataset_uri ]
+    end
+  end
+  
+  class TrainingTestValidation < ValidationExample
+    def params
+      [:algorithm_uri, :training_dataset_uri, :test_dataset_uri, :prediction_feature]
+    end
+    
+    def opt_params
+      [ :algorithm_params, :test_target_dataset_uri ]
+    end
+  end
+  
+  class SplitTestValidation < ValidationExample
+    def params
+      [:algorithm_uri, :dataset_uri, :prediction_feature]
+    end
+    
+    def opt_params
+      [ :algorithm_params, :split_ratio, :random_seed ]
+    end
+    
+    def validation_type
+      "training_test_split"
+    end
+  end
+  
+  class CrossValidation < ValidationExample
+    def params
+      [:algorithm_uri, :dataset_uri, :prediction_feature]
+    end
+    
+    def opt_params
+      [ :algorithm_params, :num_folds, :stratified, :random_seed ]
+    end
+    
+    def report_type
+      "crossvalidation"
+    end
+    
+    def validation_type
+      "crossvalidation"
+    end
+  end
+end
\ No newline at end of file