From 1c9f80e8dd360b3383cd00059b9cc181d1cca2ec Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 31 May 2011 23:29:50 +0200 Subject: test update: valiation of lazar applied to ambit datasets --- validation.rb | 146 ++++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 100 insertions(+), 46 deletions(-) diff --git a/validation.rb b/validation.rb index 02653fc..9723a33 100644 --- a/validation.rb +++ b/validation.rb @@ -22,7 +22,34 @@ class Exception end class ValidationTest < Test::Unit::TestCase - + + @@delete = true + @@feature_types = ["bbrc", "last"] + @@qmrf_test = true + @@data = [] + @@data << { :type => :crossvalidation, + :data => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?max=100", + :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/26221", + :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?max=100" } + @@data << { :type => :training_test_validation, + :train_data => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?page=0&pagesize=150", + :test_data => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?page=3&pagesize=50", + :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/26221", + :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?page=0&pagesize=150" } + @@data << { :type => :training_test_validation, + :train_data => "http://apps.ideaconsult.net:8080/ambit2/dataset/435293?page=0&pagesize=300", + :test_data => "http://apps.ideaconsult.net:8080/ambit2/dataset/435293?page=30&pagesize=10", + :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/533748", + :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/435293?page=0&pagesize=300" } + @@files = { File.new("data/hamster_carcinogenicity.csv") => :crossvalidation, + #File.new("data/hamster_carcinogenicity.mini.csv") => :crossvalidation, + #File.new("data/EPAFHM.csv") => :crossvalidation, + File.new("data/EPAFHM.mini.csv") => :crossvalidation, + File.new("data/hamster_carcinogenicity.csv") => :split_validation, + File.new("data/EPAFHM.csv") => :split_validation, + #File.new("data/StJudes-HepG2-testset_Class.csv") => :crossvalidation + } + def global_setup puts "login and upload datasets" if AA_SERVER @@ -33,39 +60,22 @@ class ValidationTest < Test::Unit::TestCase puts "AA disabled" @@subjectid = nil end - - @@delete = true - @@feature_types = ["bbrc", "last"] - @@data = [] - files = { File.new("data/hamster_carcinogenicity.csv") => :crossvalidation, - #File.new("data/hamster_carcinogenicity.mini.csv") => :crossvalidation, - #File.new("data/EPAFHM.csv") => :crossvalidation, - File.new("data/EPAFHM.mini.csv") => :crossvalidation, - File.new("data/hamster_carcinogenicity.csv") => :validation, - File.new("data/EPAFHM.csv") => :validation, - #File.new("data/StJudes-HepG2-testset_Class.csv") => :crossvalidation - } - files.each do |file,type| + @@files.each do |file,type| @@data << { :type => type, :data => ValidationTestUtil.upload_dataset(file, @@subjectid), :feat => ValidationTestUtil.prediction_feature_for_file(file), :info => file.path, :delete => true} end -# @@data << { :type => :crossvalidation, -# :data => "http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=50", -# :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/21573", -# :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=50" } -# @@data << { :type => :validation, -# :data => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?max=50", -# :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/26221", -# :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?max=50" } end def global_teardown puts "delete and logout" if @@delete - @@data.each{|data| OpenTox::Dataset.find(data[:data],@@subjectid).delete(@@subjectid) if data[:delete] and - OpenTox::Dataset.exist?(data[:data]) } + [:data, :train_data, :test_data].each do |d| + @@data.each do |data| + OpenTox::Dataset.find(data[d],@@subjectid).delete(@@subjectid) if data[d] and data[:delete] and OpenTox::Dataset.exist?(data[d]) + end + end @@vs.each{|v| v.delete(@@subjectid)} if defined?@@vs @@cvs.each{|cv| cv.delete(@@subjectid)} if defined?@@cvs @@reports.each{|report| report.delete(@@subjectid)} if defined?@@reports @@ -87,7 +97,7 @@ class ValidationTest < Test::Unit::TestCase @@vs = [] @@data.each do |data| - if data[:type]==:validation + if data[:type]==:split_validation puts "test_training_test_split "+data[:info].to_s p = { :dataset_uri => data[:data], @@ -124,6 +134,48 @@ class ValidationTest < Test::Unit::TestCase end end + + def test_training_test_validation + + @@vs = [] + @@data.each do |data| + if data[:type]==:training_test_validation + puts "test_training_test_validation "+data[:info].to_s + p = { + :training_dataset_uri => data[:train_data], + :test_dataset_uri => data[:test_data], + :algorithm_uri => File.join(CONFIG[:services]["opentox-algorithm"],"lazar"), + :algorithm_params => "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc"), + :prediction_feature => data[:feat]} + t = OpenTox::SubTask.new(nil,0,1) + def t.progress(pct) + if !defined?@last_msg or @last_msg+10 model} ) + assert v_list.size==1 and v_list.include?(v.uri) + puts v.uri unless @@delete + @@vs << v + end + end + end + + def test_validation_report #@@cv = OpenTox::Crossvalidation.find("http://local-ot/validation/crossvalidation/48", @@subjectid) @@ -314,28 +366,30 @@ class ValidationTest < Test::Unit::TestCase end end - def test_qmrf_report + if @@qmrf_test + def test_qmrf_report #@@cv = OpenTox::Crossvalidation.find("http://local-ot/validation/crossvalidation/13", @@subjectid) - - @@qmrfReports = [] - @@cvs.each do |cv| - puts "test_qmrf_report" - assert defined?cv,"no crossvalidation defined" - model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => cv.metadata[OT.dataset], :subjectid => @@subjectid}).to_s - assert model_uri.uri? -# validations = cv.metadata[OT.validation] -# assert_kind_of Array,validations -# assert validations.size==cv.metadata[OT.numFolds].to_i,validations.size.to_s+"!="+cv.metadata[OT.numFolds].to_s -# val = OpenTox::Validation.find(validations[0], @@subjectid) -# model_uri = val.metadata[OT.model] - model = OpenTox::Model::Generic.find(model_uri, @@subjectid) - assert model!=nil - #assert_rest_call_error OpenTox::NotFoundError do - # OpenTox::QMRFReport.find_for_model(model_uri, @@subjectid) - #end - qmrfReport = OpenTox::QMRFReport.create(model_uri, @@subjectid) - puts qmrfReport.uri unless @@delete - @@qmrfReports << qmrfReport + + @@qmrfReports = [] + @@cvs.each do |cv| + puts "test_qmrf_report" + assert defined?cv,"no crossvalidation defined" + model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => cv.metadata[OT.dataset], :subjectid => @@subjectid}).to_s + assert model_uri.uri? + # validations = cv.metadata[OT.validation] + # assert_kind_of Array,validations + # assert validations.size==cv.metadata[OT.numFolds].to_i,validations.size.to_s+"!="+cv.metadata[OT.numFolds].to_s + # val = OpenTox::Validation.find(validations[0], @@subjectid) + # model_uri = val.metadata[OT.model] + model = OpenTox::Model::Generic.find(model_uri, @@subjectid) + assert model!=nil + #assert_rest_call_error OpenTox::NotFoundError do + # OpenTox::QMRFReport.find_for_model(model_uri, @@subjectid) + #end + qmrfReport = OpenTox::QMRFReport.create(model_uri, @@subjectid) + puts qmrfReport.uri unless @@delete + @@qmrfReports << qmrfReport + end end end -- cgit v1.2.3