diff options
author | mguetlein <martin.guetlein@gmail.com> | 2010-05-12 10:15:34 +0200 |
---|---|---|
committer | mguetlein <martin.guetlein@gmail.com> | 2010-05-12 10:15:34 +0200 |
commit | 2679db624cdb69cb3c3d8b659d9513899c3423a4 (patch) | |
tree | ffaae64bedf7799608a53d9283ee45a188e16403 | |
parent | 3e01cb47054d16a3fe3c10a0ef5a92d886f2566f (diff) |
add model validation to nightly
-rw-r--r-- | nightly/nightly.rb | 128 | ||||
-rw-r--r-- | validation/validation_application.rb | 4 | ||||
-rw-r--r-- | validation/validation_test.rb | 21 |
3 files changed, 130 insertions, 23 deletions
diff --git a/nightly/nightly.rb b/nightly/nightly.rb index 2ad99f9..716dad4 100644 --- a/nightly/nightly.rb +++ b/nightly/nightly.rb @@ -22,6 +22,7 @@ class Nightly benchmarks = [ HamsterTrainingTestBenchmark.new, HamsterCrossvalidationBenchmark.new, MiniRegressionBenchmark.new, + CacoModelsRegressionBenchmark.new, #FatheadRegressionBenchmark.new, ] @@ -99,7 +100,7 @@ class Nightly end end - class ValidationBenchmark + class AbstractBenchmark def info_table_title return title @@ -137,12 +138,8 @@ class Nightly end - class AlgorihtmValidationBenchmark < ValidationBenchmark + class ValidationBenchmark < AbstractBenchmark - def comparable_nice_name - return "algorithm" - end - def info_table t = [] t << ["param", "uri"] @@ -150,8 +147,8 @@ class Nightly t << [k.to_s, v.to_s] end count = 1 - @algs.each do |alg| - t << ["algorithm_uri"+" ["+count.to_s+"]", alg] + @comparables.each do |alg| + t << [comparable_nice_name+" ["+count.to_s+"]", alg] count += 1 end t @@ -173,10 +170,13 @@ class Nightly raise "build report, return uri" end + def build_compare_report(comparables) + raise "build compare report, return uri" + end + def build() - raise "no algs" unless @algs + raise "no comparables" unless @comparables - @comparables = @algs @validations = Array.new(@comparables.size) @reports = Array.new(@comparables.size) @errors = {} @@ -191,12 +191,12 @@ class Nightly Thread.new do running << @comparables[i]+i.to_s begin - LOGGER.debug "Validate: "+@algs[i].to_s + LOGGER.debug "Validate: "+@comparables[i].to_s @validations[i] = validate(i) to_compare << @validations[i] if OpenTox::Utils.is_uri?(@validations[i]) begin - LOGGER.debug "Building validation-report for "+@validations[i].to_s+" ("+@algs[i].to_s+")" + LOGGER.debug "Building validation-report for "+@validations[i].to_s+" ("+@comparables[i].to_s+")" @reports[i] = build_report(i) rescue => ex LOGGER.error "validation-report error: "+ex.message @@ -221,16 +221,85 @@ class Nightly end if to_compare.size>1 - LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": build algorithm comparison report" - @comparison_report = Util.create_alg_comparison_report(to_compare) + LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": build comparison report" + @comparison_report = build_compare_report(to_compare) else LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": nothing to compare" end end end + class AlgorithmValidationBenchmark < ValidationBenchmark + + def comparable_nice_name + return "algorithm" + end + + def build() + raise "no algs" unless @algs + @comparables = @algs + super + end + + def build_compare_report(comparables) + Util.create_alg_comparison_report(comparables) + end + end + + class ModelValidationBenchmark < ValidationBenchmark + + def comparable_nice_name + return "model" + end + + def build() + raise "no models" unless @models + @comparables = @models + super + end + + def build_compare_report(comparables) + "model comparsion report not available yet" #Util.create_model_comparison_report(comparables) + end + end - class TrainingTestValidationBenchmark < AlgorihtmValidationBenchmark + class TestModelValidationBenchmark < ModelValidationBenchmark + + def info + [ training_test_info ] + end + + def training_test_info + "This is a test set validation of existing models. "+ + "The model is used to predict the test dataset. Evaluation is done by comparing the model predictions "+ + "to the actual test values (in the test (target) dataset)." + end + + def info_table_title + "Validation params" + end + + def params + p = { "test_dataset_uri" => @test_data } + p["test_target_dataset_uri"] = @test_class_data if @test_class_data + return p + end + + def validate(index) + Util.validate_model(@test_data, @test_class_data, @models[index]).to_s + end + + def build_report(index) + Util.create_report(@validations[index]) + end + + def build() + raise "no test data" unless @test_data + super + end + end + + class TrainingTestValidationBenchmark < AlgorithmValidationBenchmark def info [ training_test_info ] @@ -270,7 +339,7 @@ class Nightly end end - class CrossValidationBenchmark < AlgorihtmValidationBenchmark + class CrossValidationBenchmark < AlgorithmValidationBenchmark def info [ training_test_info ] @@ -413,6 +482,27 @@ class Nightly end end + class CacoModelsRegressionBenchmark < TestModelValidationBenchmark + + def title + "Regression model test set validation, caco dataset" + end + + def info + res = [ "Valdation of two identical(?) mlr models on caco-2 dataset." ] + super + return res + end + + def build() + @models = [ + "http://ambit.uni-plovdiv.bg:8080/ambit2/model/259260", + "http://opentox.ntua.gr:3003/model/195", + ] + @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798" + super + end + end + class FatheadRegressionBenchmark < TrainingTestValidationBenchmark def title @@ -469,6 +559,12 @@ class Nightly return uri end + def self.validate_model(test_data, test_class_data, model) + uri = OpenTox::RestClientWrapper.post @@validation_service, { :test_dataset_uri => test_data, + :test_target_dataset_uri => test_class_data, :model_uri => model } + return uri + end + def self.cross_validate_alg(data, alg, feature, folds, seed, stratified, alg_params) uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"crossvalidation"), { :dataset_uri => data, :algorithm_uri => alg, :prediction_feature => feature, :algorithm_params => alg_params, :num_folds => folds, diff --git a/validation/validation_application.rb b/validation/validation_application.rb index cfdb350..91baa72 100644 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -126,8 +126,8 @@ post '/?' do if params[:model_uri] and params[:test_dataset_uri] and !params[:training_dataset_uri] and !params[:algorithm_uri] v = Validation::Validation.new :model_uri => params[:model_uri], :test_dataset_uri => params[:test_dataset_uri], - :test_target_dataset_uri => params[:test_target_dataset_uri], - :prediction_feature => params[:prediction_feature] + :test_target_dataset_uri => params[:test_target_dataset_uri] + #:prediction_feature => params[:prediction_feature] v.validate_model elsif params[:algorithm_uri] and params[:training_dataset_uri] and params[:test_dataset_uri] and params[:prediction_feature] and !params[:model_uri] v = Validation::Validation.new :algorithm_uri => params[:algorithm_uri], diff --git a/validation/validation_test.rb b/validation/validation_test.rb index 50e93bb..cafd06a 100644 --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -33,6 +33,7 @@ class ValidationTest < Test::Unit::TestCase #ex = ex_ntua #ex = ex_ntua2 #ex = ex_tum + #ex = ex_tum2 #ex = ex_local #ex = ex_lazar_extern #ex = ex_ambit @@ -158,8 +159,10 @@ class ValidationTest < Test::Unit::TestCase #ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" #ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141" - #ex.test_data = "http://194.141.0.136:8080/compound/1" - #ex.model = "http://194.141.0.136:8080/model/414" + + ex.model = "http://ambit.uni-plovdiv.bg:8080/ambit2/model/259260" + ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798" + #ex. http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510 #ex.alg = "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/pka" #ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" @@ -167,9 +170,9 @@ class ValidationTest < Test::Unit::TestCase #ex.alg = "http://apps.ideaconsult.net:8080/ambit2/algorithm/pka" #ex.train_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/54" #53 - ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53 + #ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53 #ex.act_feat = "http://apps.ideaconsult.net:8080/ambit2/feature/22200" #22190" - ex.model = "http://apps.ideaconsult.net:8080/ambit2/model/20" + #ex.model = "http://apps.ideaconsult.net:8080/ambit2/model/20" return ex end @@ -248,7 +251,15 @@ class ValidationTest < Test::Unit::TestCase return ex end - +def ex_tum2 + ex = Example.new + ex.classification = false + ex.alg = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/J48" + ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/1027" + ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/1027" + ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/357709" + return ex + end # def test_all_validations # get '/' |