diff options
| -rw-r--r-- | data/hamster_carcinogenicity.csv | 170 | ||||
| -rw-r--r-- | fminer.rb | 15 | ||||
| -rw-r--r-- | lazar.rb | 98 | ||||
| -rw-r--r-- | validation.rb | 301 |
4 files changed, 403 insertions, 181 deletions
diff --git a/data/hamster_carcinogenicity.csv b/data/hamster_carcinogenicity.csv index d314193..52d89a3 100644 --- a/data/hamster_carcinogenicity.csv +++ b/data/hamster_carcinogenicity.csv @@ -1,86 +1,86 @@ SMILES, Hamster Carcinogenicity
-CC=O,1
-C12C3=C(C=CC=C3)CC1=CC(=CC=2)NC(C)=O,1
-O=C(N)\C(C2=CC=CO2)=C/C1=CC=C([N+]([O-])=O)O1,1
-C1(N=CNN=1)N,0
-Br(=O)(=O)[O-].[K+],1
-[Cl-].[Cd+2].[Cl-],0
-O=S(=O)([O-])[O-].[Cd+2],0
-ClC1=CC(=NC(=N1)SCC(=O)O)NC2=CC=CC(=C2C)C,0
-ClCOC,1
-C=C(Cl)C=C,0
-Clc1ccc(cc1)c2ccc(COC(C)(C)C(O)=O)cc2,0
-O=C1OC2=C(C=CC=C2)C=C1,0
-ClC(=C(C1=CC=C(C=C1)Cl)C2=CC=C(C=C2)Cl)Cl,1
-ClC(C(C1=CC=C(C=C1)Cl)C2=CC=C(C=C2)Cl)(Cl)Cl,0
-C=CCN(CC=C)N=O,1
-Cl\C2=C(/Cl)C3(Cl)C1C4CC(C1C2(Cl)C3(Cl)Cl)C5OC45,0
-O=C(N(C)C)Cl,1
-CN(C)N,1
-N(NC)C.[H]Cl.[H]Cl,1
-CCO,0
-O=C(N(CC)N=O)NCCO,1
-O=C(N(CC)N=O)NCC(=O)C,1
-C=O,0
-[O-][N+](=O)C1=CC=C(O1)C2=CSC(=N2)NNC=O,1
-O=CC1=CC=CO1,0
-OCC1CO1,1
-O=C2C1=C(OC)C=C(OC)C(Cl)=C1O[C@]32C(OC)=CC(C[C@@](C)3[H])=O,0
-ClC1=C(C(=C(C(=C1Cl)Cl)Cl)Cl)Cl,1
-NN,1
-OS(=O)(=O)O.NN,1
-CC(=O)N(O)C1=CC2=C(C=C1)C3=CC=CC=C3C2,1
-OCCNN,0
-O=C(C1=CC=NC=C1)NN,0
-OC(=O)C1=CC=NC=C1,0
-O=C(NC1=CC=CC(=C1)Cl)OC(C)C,0
-O=C(NC1=CC=CC=C1)OC(C)C,0
-[O-]C(C)=O.[O-]C(C)=O.[Pb+2].[OH-].[OH-].[Pb+2].[OH-].[OH-].[Pb+2],0
-CN(C)CCN(CC2=CC=CS2)C1=NC=CC=C1.Cl,0
-NC1=C2C(=NC(=N1)N)N=CC(=N2)CN(C3=CC=C(C=C3)C(=O)N[C@@H](CCC(=O)O)C(=O)O)C,0
-CN(N)C=O,1
-O=C(C(=C)C)OC,0
-CNN,1
-O=C(C1=CC=CN=C1)CCCN(N=O)C,0
-CC1=CC(=O)NC(=S)N1,1
-CC(C(O)=O)(OC1=CC=C(C=C1)C2CCCC3=C2C=CC=C3)C,0
-O=N[O-].[Na+],0
-[O-][N+](C1=CC=C(C2=CSC(NC(C)=O)=N2)O1)=O,1
-[O-][N+](=O)C1=CC=C(O1)C2=CSC(=N2)NC=O,1
-O=[N+](C1=CC=C2C3=C1C=CC=C3CC2)[O-],0
-N(CC(CO)O)(CC(O)C)N=O,1
-N(CC(CO)O)(CC(C)=O)N=O,1
-N(CC(CO)O)(CCO)N=O,0
-O=C(C)CN(N=O)CCO,1
-C1C(N(C(CN1N=O)C)C)C,1
-N(CC(C)=O)(CC=C)N=O,1
-N(CC(CO)O)(C)N=O,1
-O=NN1CCOCC1,1
-N1C=CC=C(C=1)C2N(N=O)CCC2,1
-C1=CC=C(C=[N+]1[O-])C2CCCN2N=O,0
-O=NN1CCCCC1,1
-O=NN1CCCC1,1
-O=C(N(CC(C)=O)N=O)NCCCl,1
-N(C(=O)N)(N=O)CC(C)=O,1
-C1(CCN=C=S)=CC=CC=C1,0
-O=C1C(C2=CC=CC=C2)(C(=O)NC(=O)N1)CC,0
-C1=C2C(=CC=C1NC3=CC=CC=C3)C=CC=C2,0
-O=C1N2C(C3=C(C=CC=C3)CC2)CN(C1)C(=O)C4CCCCC4,0
-C1(=CC(=C(O)C=C1)O)C(O)=O,0
-O=C1C2=C(C=C(C=C2O)O)O/C(=C\1O)C3=CC(=C(C=C3)O)O.O.O,0
-C1=C(C=CC(=C1)C(C2=CC=C(N)C(=C2)C)=C3C=CC(=N)C=C3)N.[H]Cl,0
-C(C1=CC=C(C=C1)N)(C2=CC=C(C=C2)N)=C3C=CC(C=C3)=N.[H]Cl,0
-OC2=CC1=C(C(O)=C2)C(C(O[C@@H]4O[C@@H]([C@H]([C@H](O)[C@H]4O)O)CO[C@H]3[C@H](O)[C@H](O)[C@H]([C@H](C)O3)O)=C(C5=CC(O)=C(C=C5)O)O1)=O,0
-ClC(=CCl)Cl,0
-NC(=O)OCC,1
-C=CCl,1
-N#[N+]C1=CC=CC=C1.F[B-](F)(F)F,0
-C1(CN(CC(N1N=O)C)N=O)C,1
-N(CCN(C)C)(C)N=O,1
-C1(CN(N=O)CC(O1)C)C,1
-O1C(N(CC1C)N=O)=O,1
-CCOC(=O)N(C)N=O,1
-C1N(COC1)N=O,1
-O=C(N(CCC1=CC=CC=C1)N=O)N,1
-O=NN1CCC1,1
-F[B-](F)(F)F.[Na+],0
+CC=O,true
+C12C3=C(C=CC=C3)CC1=CC(=CC=2)NC(C)=O,true
+O=C(N)\C(C2=CC=CO2)=C/C1=CC=C([N+]([O-])=O)O1,true
+C1(N=CNN=1)N,false
+Br(=O)(=O)[O-].[K+],true
+[Cl-].[Cd+2].[Cl-],false
+O=S(=O)([O-])[O-].[Cd+2],false
+ClC1=CC(=NC(=N1)SCC(=O)O)NC2=CC=CC(=C2C)C,false
+ClCOC,true
+C=C(Cl)C=C,false
+Clc1ccc(cc1)c2ccc(COC(C)(C)C(O)=O)cc2,false
+O=C1OC2=C(C=CC=C2)C=C1,false
+ClC(=C(C1=CC=C(C=C1)Cl)C2=CC=C(C=C2)Cl)Cl,true
+ClC(C(C1=CC=C(C=C1)Cl)C2=CC=C(C=C2)Cl)(Cl)Cl,false
+C=CCN(CC=C)N=O,true
+Cl\C2=C(/Cl)C3(Cl)C1C4CC(C1C2(Cl)C3(Cl)Cl)C5OC45,false
+O=C(N(C)C)Cl,true
+CN(C)N,true
+N(NC)C.[H]Cl.[H]Cl,true
+CCO,false
+O=C(N(CC)N=O)NCCO,true
+O=C(N(CC)N=O)NCC(=O)C,true
+C=O,false
+[O-][N+](=O)C1=CC=C(O1)C2=CSC(=N2)NNC=O,true
+O=CC1=CC=CO1,false
+OCC1CO1,true
+O=C2C1=C(OC)C=C(OC)C(Cl)=C1O[C@]32C(OC)=CC(C[C@@](C)3[H])=O,false
+ClC1=C(C(=C(C(=C1Cl)Cl)Cl)Cl)Cl,true
+NN,true
+OS(=O)(=O)O.NN,true
+CC(=O)N(O)C1=CC2=C(C=C1)C3=CC=CC=C3C2,true
+OCCNN,false
+O=C(C1=CC=NC=C1)NN,false
+OC(=O)C1=CC=NC=C1,false
+O=C(NC1=CC=CC(=C1)Cl)OC(C)C,false
+O=C(NC1=CC=CC=C1)OC(C)C,false
+[O-]C(C)=O.[O-]C(C)=O.[Pb+2].[OH-].[OH-].[Pb+2].[OH-].[OH-].[Pb+2],false
+CN(C)CCN(CC2=CC=CS2)C1=NC=CC=C1.Cl,false
+NC1=C2C(=NC(=N1)N)N=CC(=N2)CN(C3=CC=C(C=C3)C(=O)N[C@@H](CCC(=O)O)C(=O)O)C,false
+CN(N)C=O,true
+O=C(C(=C)C)OC,false
+CNN,true
+O=C(C1=CC=CN=C1)CCCN(N=O)C,false
+CC1=CC(=O)NC(=S)N1,true
+CC(C(O)=O)(OC1=CC=C(C=C1)C2CCCC3=C2C=CC=C3)C,false
+O=N[O-].[Na+],false
+[O-][N+](C1=CC=C(C2=CSC(NC(C)=O)=N2)O1)=O,true
+[O-][N+](=O)C1=CC=C(O1)C2=CSC(=N2)NC=O,true
+O=[N+](C1=CC=C2C3=C1C=CC=C3CC2)[O-],false
+N(CC(CO)O)(CC(O)C)N=O,true
+N(CC(CO)O)(CC(C)=O)N=O,true
+N(CC(CO)O)(CCO)N=O,false
+O=C(C)CN(N=O)CCO,true
+C1C(N(C(CN1N=O)C)C)C,true
+N(CC(C)=O)(CC=C)N=O,true
+N(CC(CO)O)(C)N=O,true
+O=NN1CCOCC1,true
+N1C=CC=C(C=1)C2N(N=O)CCC2,true
+C1=CC=C(C=[N+]1[O-])C2CCCN2N=O,false
+O=NN1CCCCC1,true
+O=NN1CCCC1,true
+O=C(N(CC(C)=O)N=O)NCCCl,true
+N(C(=O)N)(N=O)CC(C)=O,true
+C1(CCN=C=S)=CC=CC=C1,false
+O=C1C(C2=CC=CC=C2)(C(=O)NC(=O)N1)CC,false
+C1=C2C(=CC=C1NC3=CC=CC=C3)C=CC=C2,false
+O=C1N2C(C3=C(C=CC=C3)CC2)CN(C1)C(=O)C4CCCCC4,false
+C1(=CC(=C(O)C=C1)O)C(O)=O,false
+O=C1C2=C(C=C(C=C2O)O)O/C(=C\1O)C3=CC(=C(C=C3)O)O.O.O,false
+C1=C(C=CC(=C1)C(C2=CC=C(N)C(=C2)C)=C3C=CC(=N)C=C3)N.[H]Cl,false
+C(C1=CC=C(C=C1)N)(C2=CC=C(C=C2)N)=C3C=CC(C=C3)=N.[H]Cl,false
+OC2=CC1=C(C(O)=C2)C(C(O[C@@H]4O[C@@H]([C@H]([C@H](O)[C@H]4O)O)CO[C@H]3[C@H](O)[C@H](O)[C@H]([C@H](C)O3)O)=C(C5=CC(O)=C(C=C5)O)O1)=O,false
+ClC(=CCl)Cl,false
+NC(=O)OCC,true
+C=CCl,true
+N#[N+]C1=CC=CC=C1.F[B-](F)(F)F,false
+C1(CN(CC(N1N=O)C)N=O)C,true
+N(CCN(C)C)(C)N=O,true
+C1(CN(N=O)CC(O1)C)C,true
+O1C(N(CC1C)N=O)=O,true
+CCOC(=O)N(C)N=O,true
+C1N(COC1)N=O,true
+O=C(N(CCC1=CC=CC=C1)N=O)N,true
+O=NN1CCC1,true
+F[B-](F)(F)F.[Na+],false
@@ -33,6 +33,21 @@ class FminerTest < Test::Unit::TestCase d.delete(@@subjectid) end + def test_bbrc_rest_parameters + feature = @@classification_training_dataset.features.keys.first + #call = "curl -X POST #{File.join CONFIG[:services]["opentox-algorithm"],"fminer","bbrc"} -d \"dataset_uri=#{CGI.escape @@classification_training_dataset.uri}\" -d \"prediction_feature=#{CGI.escape feature}\" -d \"backbone=true\" -d \"min_frequency=2\"" + dataset_uri = OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-algorithm"],"fminer","bbrc"),{ + "dataset_uri" => @@classification_training_dataset.uri, + "prediction_feature" => feature, + "backbone" => true, + "min_frequency" => 2, + :subjectid => @@subjectid }) + d =OpenTox::Dataset.new dataset_uri, @@subjectid + d.load_features(@@subjectid) + assert_equal 52, d.features.size + + end + # Deactivated by AM because of efficiency problems (does not return) # def test_regression_last # feature = File.join @@regression_training_dataset.uri,"feature/LC50_mmol" @@ -1,6 +1,7 @@ require 'rubygems' require 'opentox-ruby' require 'test/unit' +require "./validate-owl.rb" class Float def round_to(x) @@ -24,26 +25,44 @@ class LazarTest < Test::Unit::TestCase =end def test_create_regression_model model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => @@regression_training_dataset.uri, :subjectid => @@subjectid}).to_s + #puts model_uri + validate_owl model_uri,@@subjectid lazar = OpenTox::Model::Lazar.find model_uri, @@subjectid @models << lazar compound = OpenTox::Compound.from_smiles("c1ccccc1NN") prediction_uri = lazar.run(:compound_uri => compound.uri, :subjectid => @@subjectid).to_s prediction = OpenTox::LazarPrediction.find(prediction_uri, @@subjectid) @predictions << prediction - #assert_equal prediction.value(compound).round_to(4), 0.3469.round_to(4) - assert_equal prediction.value(compound).round_to(4), 0.3996.round_to(4) + assert_equal prediction.value(compound).round_to(3),0.378.round_to(3) + assert_equal prediction.confidence(compound).round_to(3), 0.276.round_to(3) + #assert_equal prediction.value(compound).round_to(4), 0.2847.round_to(4) #assert_equal prediction.confidence(compound).round_to(4), 0.3223.round_to(4) - assert_equal prediction.confidence(compound).round_to(4), 0.2758.round_to(4) - #assert_equal prediction.neighbors(compound).size, 73 assert_equal prediction.neighbors(compound).size, 61 + end + + def test_create_regression_prop_model + model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => @@regression_training_dataset.uri, :subjectid => @@subjectid, :local_svm_kernel => "propositionalized"}).to_s + #puts model_uri + validate_owl model_uri,@@subjectid + lazar = OpenTox::Model::Lazar.find model_uri, @@subjectid + @models << lazar assert_equal 219, lazar.features.size - #assert_equal 185, lazar.features.size + compound = OpenTox::Compound.from_smiles("c1ccccc1NN") + prediction_uri = lazar.run(:compound_uri => compound.uri, :subjectid => @@subjectid).to_s + prediction = OpenTox::LazarPrediction.find(prediction_uri, @@subjectid) + @predictions << prediction + assert_equal prediction.value(compound).round_to(1),0.1.round_to(1) + assert_equal prediction.confidence(compound).round_to(3), 0.276.round_to(3) + #assert_equal prediction.value(compound).round_to(4), 0.2847.round_to(4) + #assert_equal prediction.confidence(compound).round_to(4), 0.3223.round_to(4) + assert_equal prediction.neighbors(compound).size, 61 end def test_classification_model # create model model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => @@classification_training_dataset.uri, :subjectid => @@subjectid}).to_s + validate_owl model_uri,@@subjectid lazar = OpenTox::Model::Lazar.find model_uri, @@subjectid @models << lazar assert_equal lazar.features.size, 52 @@ -53,7 +72,8 @@ class LazarTest < Test::Unit::TestCase prediction_uri = lazar.run(:compound_uri => compound.uri, :subjectid => @@subjectid) prediction = OpenTox::LazarPrediction.find(prediction_uri, @@subjectid) @predictions << prediction - assert_equal prediction.value(compound), false + #puts prediction_uri + assert_equal prediction.value(compound), "false" assert_equal prediction.confidence(compound).round_to(4), 0.3067.round_to(4) assert_equal prediction.neighbors(compound).size, 14 @@ -63,7 +83,7 @@ class LazarTest < Test::Unit::TestCase prediction = OpenTox::LazarPrediction.find prediction_uri, @@subjectid @predictions << prediction assert !prediction.measured_activities(compound).empty? - assert_equal prediction.measured_activities(compound).first, true + assert_equal prediction.measured_activities(compound).first.to_s, "true" assert prediction.value(compound).nil? # dataset prediction @@ -73,20 +93,78 @@ class LazarTest < Test::Unit::TestCase assert_equal prediction.compounds.size, 4 compound = OpenTox::Compound.from_smiles "CC(=Nc1ccc2c(c1)Cc1ccccc21)O" assert_equal prediction.value(compound), nil + assert_equal prediction.measured_activities(compound).first.to_s, "true" + end + + def test_classification_svm_model + + # create model + model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => @@classification_training_dataset.uri, :subjectid => @@subjectid, :prediction_algorithm => "local_svm_classification"}).to_s + lazar = OpenTox::Model::Lazar.find model_uri, @@subjectid + @models << lazar + assert_equal lazar.features.size, 52 + + # single prediction + compound = OpenTox::Compound.from_smiles("c1ccccc1NN") + prediction_uri = lazar.run(:compound_uri => compound.uri, :subjectid => @@subjectid) + prediction = OpenTox::LazarPrediction.find(prediction_uri, @@subjectid) + @predictions << prediction + assert_equal prediction.value(compound), "false" + assert_equal prediction.confidence(compound).round_to(4), 0.4131.round_to(4) + assert_equal prediction.neighbors(compound).size, 14 + + # dataset prediction + test_dataset = OpenTox::Dataset.create_from_csv_file("data/multicolumn.csv", @@subjectid) + prediction = OpenTox::LazarPrediction.find lazar.run(:dataset_uri => test_dataset.uri, :subjectid => @@subjectid), @@subjectid + @predictions << prediction + assert_equal prediction.compounds.size, 4 + compound = OpenTox::Compound.from_smiles "CC(=Nc1ccc2c(c1)Cc1ccccc21)O" + assert_equal prediction.value(compound), nil + assert_equal prediction.measured_activities(compound).first, true + end + + def test_classification_svm_prop_model + + + # create model + model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => @@classification_training_dataset.uri, :subjectid => @@subjectid, :prediction_algorithm => "local_svm_classification", :local_svm_kernel => "propositionalized"}).to_s + lazar = OpenTox::Model::Lazar.find model_uri, @@subjectid + @models << lazar + assert_equal lazar.features.size, 52 + + # single prediction + compound = OpenTox::Compound.from_smiles("c1ccccc1NN") + prediction_uri = lazar.run(:compound_uri => compound.uri, :subjectid => @@subjectid) + prediction = OpenTox::LazarPrediction.find(prediction_uri, @@subjectid) + @predictions << prediction + assert_equal prediction.value(compound), "false" + assert_equal prediction.confidence(compound).round_to(4), 0.4131.round_to(4) + assert_equal prediction.neighbors(compound).size, 14 + + # dataset prediction + test_dataset = OpenTox::Dataset.create_from_csv_file("data/multicolumn.csv", @@subjectid) + prediction = OpenTox::LazarPrediction.find lazar.run(:dataset_uri => test_dataset.uri, :subjectid => @@subjectid), @@subjectid + @predictions << prediction + assert_equal prediction.compounds.size, 4 + compound = OpenTox::Compound.from_smiles "CC(=Nc1ccc2c(c1)Cc1ccccc21)O" + assert_equal prediction.value(compound), nil assert_equal prediction.measured_activities(compound).first, true + end =begin def test_ambit_classification_model # create model - dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/9" + dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=400" feature_uri ="http://apps.ideaconsult.net:8080/ambit2/feature/21573" #model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => dataset_uri, :prediction_feature => feature_uri}).to_s #lazar = OpenTox::Model::Lazar.find model_uri model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => dataset_uri, :prediction_feature => feature_uri, :subjectid => @@subjectid}).to_s + validate_owl model_uri,@@subjectid lazar = OpenTox::Model::Lazar.find model_uri, @@subjectid - assert_equal lazar.features.size, 6609 + puts lazar.features.size + assert_equal lazar.features.size, 1874 #puts "Model: #{lazar.uri}" #puts lazar.features.size @@ -98,7 +176,7 @@ class LazarTest < Test::Unit::TestCase prediction = OpenTox::LazarPrediction.find(prediction_uri, @@subjectid) #puts "Prediction: #{prediction.uri}" #puts prediction.value(compound) - assert_equal prediction.value(compound), true + assert_equal prediction.value(compound), "3.0" #puts @prediction.confidence(compound).round_to(4) #assert_equal @prediction.confidence(compound).round_to(4), 0.3005.round_to(4) #assert_equal @prediction.neighbors(compound).size, 15 diff --git a/validation.rb b/validation.rb index 14ef5ee..ff0725d 100644 --- a/validation.rb +++ b/validation.rb @@ -22,7 +22,34 @@ class Exception end class ValidationTest < Test::Unit::TestCase - + + @@delete = true + @@feature_types = ["bbrc", "last"] + @@qmrf_test = true + @@data = [] + @@data << { :type => :crossvalidation, + :data => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?max=100", + :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/26221", + :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?max=100" } + @@data << { :type => :training_test_validation, + :train_data => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?page=0&pagesize=150", + :test_data => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?page=3&pagesize=50", + :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/26221", + :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?page=0&pagesize=150" } + @@data << { :type => :training_test_validation, + :train_data => "http://apps.ideaconsult.net:8080/ambit2/dataset/435293?page=0&pagesize=300", + :test_data => "http://apps.ideaconsult.net:8080/ambit2/dataset/435293?page=30&pagesize=10", + :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/533748", + :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/435293?page=0&pagesize=300" } + @@files = { File.new("data/hamster_carcinogenicity.csv") => :crossvalidation, + #File.new("data/hamster_carcinogenicity.mini.csv") => :crossvalidation, + #File.new("data/EPAFHM.csv") => :crossvalidation, + File.new("data/EPAFHM.mini.csv") => :crossvalidation, + File.new("data/hamster_carcinogenicity.csv") => :split_validation, + File.new("data/EPAFHM.csv") => :split_validation, + #File.new("data/StJudes-HepG2-testset_Class.csv") => :crossvalidation + } + def global_setup puts "login and upload datasets" if AA_SERVER @@ -33,37 +60,27 @@ class ValidationTest < Test::Unit::TestCase puts "AA disabled" @@subjectid = nil end - - @@data = [] - files = { File.new("data/hamster_carcinogenicity.mini.csv") => :crossvalidation, - File.new("data/EPAFHM.mini.csv") => :crossvalidation, - File.new("data/hamster_carcinogenicity.csv") => :validation, - File.new("data/EPAFHM.csv") => :validation, -# File.new("data/StJudes-HepG2-testset_Class.csv") => :crossvalidation - } - files.each do |file,type| + @@files.each do |file,type| @@data << { :type => type, :data => ValidationTestUtil.upload_dataset(file, @@subjectid), :feat => ValidationTestUtil.prediction_feature_for_file(file), :info => file.path, :delete => true} end -# @@data << { :type => :crossvalidation, -# :data => "http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=50", -# :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/21573", -# :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=50" } -# @@data << { :type => :validation, -# :data => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?max=50", -# :feat => "http://apps.ideaconsult.net:8080/ambit2/feature/26221", -# :info => "http://apps.ideaconsult.net:8080/ambit2/dataset/272?max=50" } end def global_teardown puts "delete and logout" - @@data.each{|data| OpenTox::Dataset.find(data[:data],@@subjectid).delete(@@subjectid) if data[:delete]} - @@vs.each{|v| v.delete(@@subjectid)} if defined?@@vs - @@cvs.each{|cv| cv.delete(@@subjectid)} if defined?@@cvs - @@reports.each{|report| report.delete(@@subjectid)} if defined?@@reports - @@qmrfReports.each{|qmrfReport| qmrfReport.delete(@@subjectid)} if defined?@@qmrfReports + if @@delete + [:data, :train_data, :test_data].each do |d| + @@data.each do |data| + OpenTox::Dataset.find(data[d],@@subjectid).delete(@@subjectid) if data[d] and data[:delete] and OpenTox::Dataset.exist?(data[d]) + end + end + @@vs.each{|v| v.delete(@@subjectid)} if defined?@@vs + @@cvs.each{|cv| cv.delete(@@subjectid)} if defined?@@cvs + @@reports.each{|report| report.delete(@@subjectid)} if defined?@@reports + @@qmrfReports.each{|qmrfReport| qmrfReport.delete(@@subjectid)} if defined?@@qmrfReports + end OpenTox::Authorization.logout(@@subjectid) if AA_SERVER end @@ -80,7 +97,7 @@ class ValidationTest < Test::Unit::TestCase @@vs = [] @@data.each do |data| - if data[:type]==:validation + if data[:type]==:split_validation puts "test_training_test_split "+data[:info].to_s p = { :dataset_uri => data[:data], @@ -91,7 +108,7 @@ class ValidationTest < Test::Unit::TestCase :random_seed => 2} t = OpenTox::SubTask.new(nil,0,1) def t.progress(pct) - if !defined?@last_msg or @last_msg+3<Time.new + if !defined?@last_msg or @last_msg+10<Time.new puts "waiting for training-test-split validation: "+pct.to_s @last_msg=Time.new end @@ -111,11 +128,54 @@ class ValidationTest < Test::Unit::TestCase assert model.uri? v_list = OpenTox::Validation.list( {:model => model} ) assert v_list.size==1 and v_list.include?(v.uri) + puts v.uri unless @@delete @@vs << v end end end + + def test_training_test_validation + + @@vs = [] + @@data.each do |data| + if data[:type]==:training_test_validation + puts "test_training_test_validation "+data[:info].to_s + p = { + :training_dataset_uri => data[:train_data], + :test_dataset_uri => data[:test_data], + :algorithm_uri => File.join(CONFIG[:services]["opentox-algorithm"],"lazar"), + :algorithm_params => "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc"), + :prediction_feature => data[:feat]} + t = OpenTox::SubTask.new(nil,0,1) + def t.progress(pct) + if !defined?@last_msg or @last_msg+10<Time.new + puts "waiting for training-test-set validation: "+pct.to_s + @last_msg=Time.new + end + end + def t.waiting_for(task_uri); end + v = OpenTox::Validation.create_training_test_validation(p, @@subjectid, t) + assert v.uri.uri? + if @@subjectid + assert_rest_call_error OpenTox::NotAuthorizedError do + OpenTox::Validation.find(v.uri) + end + end + v = OpenTox::Validation.find(v.uri, @@subjectid) + assert_valid_date v + assert v.uri.uri? + model = v.metadata[OT.model] + assert model.uri? + v_list = OpenTox::Validation.list( {:model => model} ) + assert v_list.size==1 and v_list.include?(v.uri) + puts v.uri unless @@delete + @@vs << v + end + end + end + + def test_validation_report #@@cv = OpenTox::Crossvalidation.find("http://local-ot/validation/crossvalidation/48", @@subjectid) @@ -145,6 +205,7 @@ class ValidationTest < Test::Unit::TestCase assert_equal report.uri,report2.uri report3_uri = v.find_or_create_report(@@subjectid) assert_equal report.uri,report3_uri + puts report2.uri unless @@delete @@reports << report2 end end @@ -164,51 +225,69 @@ class ValidationTest < Test::Unit::TestCase # OpenTox::Crossvalidation.find(File.join(CONFIG[:services]["opentox-validation"],"crossvalidation/noexistingid")) #end @@cvs = [] + @@cv_datasets = [] + @@cv_identifiers = [] @@data.each do |data| if data[:type]==:crossvalidation - puts "test_crossvalidation "+data[:info].to_s - p = { - :dataset_uri => data[:data], - :algorithm_uri => File.join(CONFIG[:services]["opentox-algorithm"],"lazar"), - :algorithm_params => "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc"), - :prediction_feature => data[:feat], - :num_folds => 2 } - t = OpenTox::SubTask.new(nil,0,1) - def t.progress(pct) - if !defined?@last_msg or @last_msg+3<Time.new - puts "waiting for crossvalidation: "+pct.to_s - @last_msg=Time.new + @@feature_types.each do |fminer| + puts "test_crossvalidation "+data[:info].to_s+" "+fminer + p = { + :dataset_uri => data[:data], + :algorithm_uri => File.join(CONFIG[:services]["opentox-algorithm"],"lazar"), + :algorithm_params => "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/"+fminer)+ + (data[:info] =~ /mini/ ? ";backbone=false;min_chisq_significance=0.0" : ""), + :prediction_feature => data[:feat], + :num_folds => 10 } + #:num_folds => 2 } + t = OpenTox::SubTask.new(nil,0,1) + def t.progress(pct) + if !defined?@last_msg or @last_msg+10<Time.new + puts "waiting for crossvalidation: "+pct.to_s + @last_msg=Time.new + end end - end - def t.waiting_for(task_uri); end - cv = OpenTox::Crossvalidation.create(p, @@subjectid, t) - assert cv.uri.uri? - if @@subjectid - assert_rest_call_error OpenTox::NotAuthorizedError do - OpenTox::Crossvalidation.find(cv.uri) + def t.waiting_for(task_uri); end + cv = OpenTox::Crossvalidation.create(p, @@subjectid, t) + assert cv.uri.uri? + if @@subjectid + assert_rest_call_error OpenTox::NotAuthorizedError do + OpenTox::Crossvalidation.find(cv.uri) + end end - end - cv = OpenTox::Crossvalidation.find(cv.uri, @@subjectid) - assert_valid_date cv - assert cv.uri.uri? - if @@subjectid - assert_rest_call_error OpenTox::NotAuthorizedError do - cv.summary(cv) + cv = OpenTox::Crossvalidation.find(cv.uri, @@subjectid) + assert_valid_date cv + assert cv.uri.uri? + if @@subjectid + assert_rest_call_error OpenTox::NotAuthorizedError do + cv.summary(cv) + end end + summary = cv.summary(@@subjectid) + assert_kind_of Hash,summary + + algorithm = cv.metadata[OT.algorithm] + assert algorithm.uri? + cv_list = OpenTox::Crossvalidation.list( {:algorithm => algorithm} ) + assert cv_list.include?(cv.uri) + cv_list.each do |cv_uri| + begin + alg = OpenTox::Crossvalidation.find(cv_uri, @@subjectid).metadata[OT.algorithm] + assert alg==algorithm,"wrong algorithm for filtered crossvalidation, should be: '"+algorithm.to_s+"', is: '"+alg.to_s+"'" + rescue OpenTox::RestCallError => e + raise "error Report of RestCallError is no errorReport: "+e.errorCause.class.to_s+":\n"+e.errorCause.to_yaml unless e.errorCause.is_a?(OpenTox::ErrorReport) + report = e.errorCause + while report.errorCause + report = report.errorCause + end + assert_equal report.errorType,OpenTox::NotAuthorizedError.to_s + end + end + puts cv.uri unless @@delete + + @@cvs << cv + @@cv_datasets << data + @@cv_identifiers << fminer end - summary = cv.summary(@@subjectid) - assert_kind_of Hash,summary - - algorithm = cv.metadata[OT.algorithm] - assert algorithm.uri? - cv_list = OpenTox::Crossvalidation.list( {:algorithm => algorithm} ) - assert cv_list.include?(cv.uri) - cv_list.each do |cv_uri| - alg = OpenTox::Crossvalidation.find(cv_uri).metadata[OT.algorithm] - assert alg==algorithm,"wrong algorithm for filtered crossvalidation, should be: '"+algorithm.to_s+"', is: '"+alg.to_s+"'" - end - - @@cvs << cv end end end @@ -229,7 +308,7 @@ class ValidationTest < Test::Unit::TestCase OpenTox::CrossvalidationReport.create(cv.uri) end end - assert OpenTox::ValidationReport.find_for_validation(cv.uri,@@subjectid)==nil + assert OpenTox::CrossvalidationReport.find_for_crossvalidation(cv.uri,@@subjectid)==nil report = OpenTox::CrossvalidationReport.create(cv.uri,@@subjectid) assert report.uri.uri? if @@subjectid @@ -244,32 +323,81 @@ class ValidationTest < Test::Unit::TestCase assert_equal report.uri,report2.uri report3_uri = cv.find_or_create_report(@@subjectid) assert_equal report.uri,report3_uri + puts report2.uri unless @@delete @@reports << report2 end end - def test_qmrf_report + def test_crossvalidation_compare_report + @@reports = [] unless defined?@@reports + @@cvs.size.times do |i| + @@cvs.size.times do |j| + if j>i and @@cv_datasets[i]==@@cv_datasets[j] + puts "test_crossvalidation_compare_report" + assert_kind_of OpenTox::Crossvalidation,@@cvs[i] + assert_kind_of OpenTox::Crossvalidation,@@cvs[j] + hash = { @@cv_identifiers[i] => [@@cvs[i].uri], + @@cv_identifiers[j] => [@@cvs[j].uri] } + if @@subjectid + assert_rest_call_error OpenTox::NotAuthorizedError do + OpenTox::AlgorithmComparisonReport.create hash,@@subjectid + end + end + assert OpenTox::AlgorithmComparisonReport.find_for_crossvalidation(@@cvs[i].uri,@@subjectid)==nil + assert OpenTox::AlgorithmComparisonReport.find_for_crossvalidation(@@cvs[j].uri,@@subjectid)==nil + report = OpenTox::AlgorithmComparisonReport.create hash,@@subjectid + assert report.uri.uri? + if @@subjectid + assert_rest_call_error OpenTox::NotAuthorizedError do + OpenTox::AlgorithmComparisonReport.find(report.uri) + end + end + report = OpenTox::AlgorithmComparisonReport.find(report.uri,@@subjectid) + assert_valid_date report + assert report.uri.uri? + report2 = OpenTox::AlgorithmComparisonReport.find_for_crossvalidation(@@cvs[i].uri,@@subjectid) + assert_equal report.uri,report2.uri + report3 = OpenTox::AlgorithmComparisonReport.find_for_crossvalidation(@@cvs[j].uri,@@subjectid) + assert_equal report.uri,report3.uri + puts report2.uri unless @@delete + @@reports << report2 + end + end + end + end + + if @@qmrf_test + def test_qmrf_report #@@cv = OpenTox::Crossvalidation.find("http://local-ot/validation/crossvalidation/13", @@subjectid) - - @@qmrfReports = [] - @@cvs.each do |cv| - puts "test_qmrf_report" - assert defined?cv,"no crossvalidation defined" - validations = cv.metadata[OT.validation] - assert_kind_of Array,validations - assert validations.size==cv.metadata[OT.numFolds].to_i,validations.size.to_s+"!="+cv.metadata[OT.numFolds].to_s - val = OpenTox::Validation.find(validations[0], @@subjectid) - model_uri = val.metadata[OT.model] - - model = OpenTox::Model::Generic.find(model_uri, @@subjectid) - assert model!=nil - - #assert_rest_call_error OpenTox::NotFoundError do - # OpenTox::QMRFReport.find_for_model(model_uri, @@subjectid) - #end - - @@qmrfReports << OpenTox::QMRFReport.create(model_uri, @@subjectid) + + @@qmrfReports = [] + @@cvs.each do |cv| + puts "test_qmrf_report" + assert defined?cv,"no crossvalidation defined" + assert cv.metadata[OT.validation].is_a?(Array) + assert cv.metadata[OT.validation].first.uri? + validation = OpenTox::Validation.find(cv.metadata[OT.validation].first) + prediction_feature_uri = validation.metadata[OT.predictionFeature] + assert prediction_feature_uri.uri? + model_uri = OpenTox::Algorithm::Lazar.new.run({:dataset_uri => cv.metadata[OT.dataset], :prediction_feature => prediction_feature_uri, + :subjectid => @@subjectid}).to_s + assert model_uri.uri? + # validations = cv.metadata[OT.validation] + # assert_kind_of Array,validations + # assert validations.size==cv.metadata[OT.numFolds].to_i,validations.size.to_s+"!="+cv.metadata[OT.numFolds].to_s + # val = OpenTox::Validation.find(validations[0], @@subjectid) + # model_uri = val.metadata[OT.model] + model = OpenTox::Model::Generic.find(model_uri, @@subjectid) + assert model!=nil + #assert_rest_call_error OpenTox::NotFoundError do + # OpenTox::QMRFReport.find_for_model(model_uri, @@subjectid) + #end + qmrfReport = OpenTox::QMRFReport.create(model_uri, @@subjectid) + puts qmrfReport.uri unless @@delete + @@qmrfReports << qmrfReport + end end + end ################### utils and overrides ########################## @@ -302,8 +430,9 @@ class ValidationTest < Test::Unit::TestCase raise "no opentox object" unless opentox_object.class.to_s.split("::").first=="OpenTox" assert opentox_object.metadata.is_a?(Hash) assert opentox_object.metadata[DC.date].to_s.length>0,"date not set for "+opentox_object.uri.to_s+", is metadata loaded? (use find)" -=begin time = Time.parse(opentox_object.metadata[DC.date]) + assert time!=nil +=begin assert time<Time.new,"date of "+opentox_object.uri.to_s+" is in the future: "+time.to_s assert time>Time.new-(10*60),opentox_object.uri.to_s+" took longer than 10 minutes "+time.to_s =end |