From 3e091b1934e824f2e63266aa86545ad63a205da9 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 7 Dec 2011 16:29:43 +0100 Subject: add tests for validation probs --- validation.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'validation.rb') diff --git a/validation.rb b/validation.rb index 06254c3..7f422cc 100644 --- a/validation.rb +++ b/validation.rb @@ -120,6 +120,7 @@ class ValidationTest < Test::Unit::TestCase v = OpenTox::Validation.find(v.uri, @@subjectid) assert_valid_date v assert v.uri.uri? + assert_prob_correct(v) model = v.metadata[OT.model] assert model.uri? v_list = OpenTox::Validation.list( {:model => model} ) @@ -161,6 +162,7 @@ class ValidationTest < Test::Unit::TestCase v = OpenTox::Validation.find(v.uri, @@subjectid) assert_valid_date v assert v.uri.uri? + assert_prob_correct(v) model = v.metadata[OT.model] assert model.uri? v_list = OpenTox::Validation.list( {:model => model} ) @@ -260,6 +262,7 @@ class ValidationTest < Test::Unit::TestCase end stats_val = cv.statistics(@@subjectid) assert_kind_of OpenTox::Validation,stats_val + assert_prob_correct(stats_val) algorithm = cv.metadata[OT.algorithm] assert algorithm.uri? @@ -441,6 +444,18 @@ class ValidationTest < Test::Unit::TestCase assert time>Time.new-(10*60),opentox_object.uri.to_s+" took longer than 10 minutes "+time.to_s =end end + + def assert_prob_correct( validation ) + class_stats = validation.metadata[OT.classificationStatistics] + if class_stats != nil + class_value_stats = class_stats[OT.classValueStatistics] + class_value_stats.each do |cs| + #puts cs[OT.positivePredictiveValue] + #puts validation.probabilities(0,cs[OT.classValue]).inspect + assert cs[OT.positivePredictiveValue]==validation.probabilities(0,cs[OT.classValue])[:probs][cs[OT.classValue]] + end + end + end # hack to have a global_setup and global_teardown def teardown @@ -457,4 +472,3 @@ class ValidationTest < Test::Unit::TestCase end - -- cgit v1.2.3 From 0017a1ea3abc5588c0a90adc158ccf95404effaf Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 8 Dec 2011 11:07:47 +0100 Subject: add missing subjectid for validation test --- validation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'validation.rb') diff --git a/validation.rb b/validation.rb index 7f422cc..caea23c 100644 --- a/validation.rb +++ b/validation.rb @@ -452,7 +452,7 @@ class ValidationTest < Test::Unit::TestCase class_value_stats.each do |cs| #puts cs[OT.positivePredictiveValue] #puts validation.probabilities(0,cs[OT.classValue]).inspect - assert cs[OT.positivePredictiveValue]==validation.probabilities(0,cs[OT.classValue])[:probs][cs[OT.classValue]] + assert cs[OT.positivePredictiveValue]==validation.probabilities(0,cs[OT.classValue],@@subjectid)[:probs][cs[OT.classValue]] end end end -- cgit v1.2.3 From 2ec47d50182fe15420a17d4db05f62fed286f1b4 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 13 Dec 2011 11:19:14 +0100 Subject: enabling compare test again, add params to report creation --- validation.rb | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'validation.rb') diff --git a/validation.rb b/validation.rb index caea23c..849cbf6 100644 --- a/validation.rb +++ b/validation.rb @@ -19,8 +19,7 @@ end class ValidationTest < Test::Unit::TestCase @@delete = true - #@@feature_types = ["bbrc", "last"] - @@feature_types = ["bbrc"] + @@hamster_cv_feature_types = ["bbrc", "last"] @@qmrf_test = true @@data = [] @@data << { :type => :crossvalidation, @@ -37,7 +36,8 @@ class ValidationTest < Test::Unit::TestCase :test_data => "http://apps.ideaconsult.net:8080/ambit2/dataset/435293?page=30&pagesize=10", :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/533748", :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/435293?page=0&pagesize=300" } - @@files = { File.new("data/hamster_carcinogenicity.csv") => :crossvalidation, + @@files = { + File.new("data/hamster_carcinogenicity.csv") => :crossvalidation, #File.new("data/hamster_carcinogenicity.mini.csv") => :crossvalidation, #File.new("data/EPAFHM.csv") => :crossvalidation, File.new("data/EPAFHM.mini.csv") => :crossvalidation, @@ -91,7 +91,7 @@ class ValidationTest < Test::Unit::TestCase def test_training_test_split - @@vs = [] + @@vs = [] unless defined?@@vs @@data.each do |data| if data[:type]==:split_validation puts "test_training_test_split "+data[:info].to_s @@ -134,7 +134,7 @@ class ValidationTest < Test::Unit::TestCase def test_training_test_validation - @@vs = [] + @@vs = [] unless defined?@@vs @@data.each do |data| if data[:type]==:training_test_validation puts "test_training_test_validation "+data[:info].to_s @@ -189,7 +189,8 @@ class ValidationTest < Test::Unit::TestCase end report = OpenTox::ValidationReport.find_for_validation(v.uri,@@subjectid) assert report==nil,"report already exists for validation\nreport: "+(report ? report.uri.to_s : "")+"\nvalidation: "+v.uri.to_s - report = OpenTox::ValidationReport.create(v.uri,@@subjectid) + params = {:min_confidence => 0.05} + report = OpenTox::ValidationReport.create(v.uri,params,@@subjectid) assert report.uri.uri? if @@subjectid assert_rest_call_error OpenTox::NotAuthorizedError do @@ -227,7 +228,8 @@ class ValidationTest < Test::Unit::TestCase @@cv_identifiers = [] @@data.each do |data| if data[:type]==:crossvalidation - @@feature_types.each do |fminer| + @@hamster_cv_feature_types.each do |fminer| + next unless (fminer==@@hamster_cv_feature_types[0] or data[:info].to_s =~ /hamster_carcinogenicity.csv/) puts "test_crossvalidation "+data[:info].to_s+" "+fminer p = { :dataset_uri => data[:data], @@ -344,7 +346,9 @@ class ValidationTest < Test::Unit::TestCase end assert OpenTox::AlgorithmComparisonReport.find_for_crossvalidation(@@cvs[i].uri,@@subjectid)==nil assert OpenTox::AlgorithmComparisonReport.find_for_crossvalidation(@@cvs[j].uri,@@subjectid)==nil - report = OpenTox::AlgorithmComparisonReport.create hash,@@subjectid + + params = {:ttest_significance => 0.95, :ttest_attributes => "real_runtime,percent_unpredicted", :max_num_predictions => 5} + report = OpenTox::AlgorithmComparisonReport.create hash,params,@@subjectid assert report.uri.uri? if @@subjectid assert_rest_call_error OpenTox::NotAuthorizedError do -- cgit v1.2.3 From 192a31f95cb40aebd8100039295fd748407080f0 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 3 Jan 2012 13:37:23 +0100 Subject: fix assert-rest-call-error method --- validation.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'validation.rb') diff --git a/validation.rb b/validation.rb index 849cbf6..b608d40 100644 --- a/validation.rb +++ b/validation.rb @@ -188,7 +188,7 @@ class ValidationTest < Test::Unit::TestCase end end report = OpenTox::ValidationReport.find_for_validation(v.uri,@@subjectid) - assert report==nil,"report already exists for validation\nreport: "+(report ? report.uri.to_s : "")+"\nvalidation: "+v.uri.to_s + assert_nil report,"report already exists for validation\nreport: "+(report ? report.uri.to_s : "")+"\nvalidation: "+v.uri.to_s params = {:min_confidence => 0.05} report = OpenTox::ValidationReport.create(v.uri,params,@@subjectid) assert report.uri.uri? @@ -425,6 +425,7 @@ class ValidationTest < Test::Unit::TestCase end begin yield + assert false,"no rest-call error thrown" rescue OpenTox::RestCallError => e raise "error Report of RestCallError is no errorReport: "+e.errorCause.class.to_s+":\n"+e.errorCause.to_yaml unless e.errorCause.is_a?(OpenTox::ErrorReport) report = e.errorCause -- cgit v1.2.3 From 3001832049c588dc859fb5b8671d97eb0276e650 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 3 Jan 2012 14:37:22 +0100 Subject: replaced @@subjectid with AA_SERVER when checking for aa-enabled --- validation.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'validation.rb') diff --git a/validation.rb b/validation.rb index b608d40..83f8bfe 100644 --- a/validation.rb +++ b/validation.rb @@ -112,7 +112,7 @@ class ValidationTest < Test::Unit::TestCase def t.waiting_for(task_uri); end v = OpenTox::Validation.create_training_test_split(p, @@subjectid, t) assert v.uri.uri? - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do OpenTox::Validation.find(v.uri) end @@ -154,7 +154,7 @@ class ValidationTest < Test::Unit::TestCase def t.waiting_for(task_uri); end v = OpenTox::Validation.create_training_test_validation(p, @@subjectid, t) assert v.uri.uri? - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do OpenTox::Validation.find(v.uri) end @@ -182,7 +182,7 @@ class ValidationTest < Test::Unit::TestCase puts "test_validation_report" assert defined?v,"no validation defined" assert_kind_of OpenTox::Validation,v - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do OpenTox::ValidationReport.create(v.uri) end @@ -192,7 +192,7 @@ class ValidationTest < Test::Unit::TestCase params = {:min_confidence => 0.05} report = OpenTox::ValidationReport.create(v.uri,params,@@subjectid) assert report.uri.uri? - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do OpenTox::ValidationReport.find(report.uri) end @@ -249,7 +249,7 @@ class ValidationTest < Test::Unit::TestCase def t.waiting_for(task_uri); end cv = OpenTox::Crossvalidation.create(p, @@subjectid, t) assert cv.uri.uri? - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do OpenTox::Crossvalidation.find(cv.uri) end @@ -257,7 +257,7 @@ class ValidationTest < Test::Unit::TestCase cv = OpenTox::Crossvalidation.find(cv.uri, @@subjectid) assert_valid_date cv assert cv.uri.uri? - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do cv.statistics(cv) end @@ -304,7 +304,7 @@ class ValidationTest < Test::Unit::TestCase #assert_rest_call_error OpenTox::NotFoundError do # OpenTox::CrossvalidationReport.find_for_crossvalidation(cv.uri) #end - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do OpenTox::CrossvalidationReport.create(cv.uri) end @@ -312,7 +312,7 @@ class ValidationTest < Test::Unit::TestCase assert OpenTox::CrossvalidationReport.find_for_crossvalidation(cv.uri,@@subjectid)==nil report = OpenTox::CrossvalidationReport.create(cv.uri,@@subjectid) assert report.uri.uri? - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do OpenTox::CrossvalidationReport.find(report.uri) end @@ -339,7 +339,7 @@ class ValidationTest < Test::Unit::TestCase assert_kind_of OpenTox::Crossvalidation,@@cvs[j] hash = { @@cv_identifiers[i] => [@@cvs[i].uri], @@cv_identifiers[j] => [@@cvs[j].uri] } - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do OpenTox::AlgorithmComparisonReport.create hash,@@subjectid end @@ -350,7 +350,7 @@ class ValidationTest < Test::Unit::TestCase params = {:ttest_significance => 0.95, :ttest_attributes => "real_runtime,percent_unpredicted", :max_num_predictions => 5} report = OpenTox::AlgorithmComparisonReport.create hash,params,@@subjectid assert report.uri.uri? - if @@subjectid + if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do OpenTox::AlgorithmComparisonReport.find(report.uri) end @@ -419,7 +419,7 @@ class ValidationTest < Test::Unit::TestCase # checks RestCallError type def assert_rest_call_error( ex ) - if ex==OpenTox::NotAuthorizedError and @@subjectid==nil + if ex==OpenTox::NotAuthorizedError and AA_SERVER==nil puts "AA disabled: skipping test for not authorized" return end -- cgit v1.2.3 From 3245c5813517f6e6754b3263dbe816f66279727f Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 5 Jan 2012 14:21:54 +0100 Subject: fix validation-comparison-test with aa --- validation.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'validation.rb') diff --git a/validation.rb b/validation.rb index 83f8bfe..919a562 100644 --- a/validation.rb +++ b/validation.rb @@ -341,7 +341,7 @@ class ValidationTest < Test::Unit::TestCase @@cv_identifiers[j] => [@@cvs[j].uri] } if AA_SERVER assert_rest_call_error OpenTox::NotAuthorizedError do - OpenTox::AlgorithmComparisonReport.create hash,@@subjectid + OpenTox::AlgorithmComparisonReport.create hash,{} end end assert OpenTox::AlgorithmComparisonReport.find_for_crossvalidation(@@cvs[i].uri,@@subjectid)==nil @@ -427,7 +427,8 @@ class ValidationTest < Test::Unit::TestCase yield assert false,"no rest-call error thrown" rescue OpenTox::RestCallError => e - raise "error Report of RestCallError is no errorReport: "+e.errorCause.class.to_s+":\n"+e.errorCause.to_yaml unless e.errorCause.is_a?(OpenTox::ErrorReport) + raise "error Report of RestCallError is no errorReport: "+ + e.errorCause.class.to_s+":\n"+e.errorCause.to_yaml unless e.errorCause.is_a?(OpenTox::ErrorReport) report = e.errorCause while report.errorCause report = report.errorCause -- cgit v1.2.3 From c5eb2319eaedfb289d02efea43019008500e7511 Mon Sep 17 00:00:00 2001 From: rautenberg Date: Tue, 13 Mar 2012 10:38:07 +0100 Subject: add missing subjectid --- validation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'validation.rb') diff --git a/validation.rb b/validation.rb index 919a562..159bf8e 100644 --- a/validation.rb +++ b/validation.rb @@ -69,7 +69,7 @@ class ValidationTest < Test::Unit::TestCase if @@delete [:data, :train_data, :test_data].each do |d| @@data.each do |data| - OpenTox::Dataset.find(data[d],@@subjectid).delete(@@subjectid) if data[d] and data[:delete] and OpenTox::Dataset.exist?(data[d]) + OpenTox::Dataset.find(data[d],@@subjectid).delete(@@subjectid) if data[d] and data[:delete] and OpenTox::Dataset.exist?(data[d], @@subjectid) end end @@vs.each{|v| v.delete(@@subjectid)} if defined?@@vs -- cgit v1.2.3