diff options
Diffstat (limited to 'test')
| -rwxr-xr-x[-rw-r--r--] | test/test_application.rb | 19 | ||||
| -rwxr-xr-x[-rw-r--r--] | test/test_examples.rb | 382 | ||||
| -rwxr-xr-x[-rw-r--r--] | test/test_examples_util.rb | 231 | ||||
| -rw-r--r-- | test/unit_test.rb | 189 |
4 files changed, 742 insertions, 79 deletions
diff --git a/test/test_application.rb b/test/test_application.rb index cbc1057..f255d1d 100644..100755 --- a/test/test_application.rb +++ b/test/test_application.rb @@ -2,16 +2,23 @@ post '/test_validation/?' do validationExamples = ValidationExamples.select(params[:select]) return "please \"select\" a single validation example:\n"+ValidationExamples.list if validationExamples.size!=1 or validationExamples[0].size!=1 - OpenTox::Task.as_task do + task = OpenTox::Task.create("Test validation",url_for("/test_validation",:full)) do |task| v = validationExamples[0][0] ex = v.new + ex.subjectid = @subjectid ex.upload_files + task.progress(10) ex.check_requirements - ex.validate - raise ex.validation_error if ex.validation_error - ex.report if params[:report] - raise ex.report_error if ex.report_error - ex.validation_uri + (params[:report] ? ","+ex.report_uri : "") + ex.validate(OpenTox::SubTask.create(task,10,80)) + #raise ex.validation_error if ex.validation_error + ex.report(OpenTox::SubTask.create(task,80,99)) unless params[:report]=="false" + #raise ex.report_error if ex.report_error + if ex.report_uri + ex.report_uri + else + ex.validation_uri + end end + return_task(task) end diff --git a/test/test_examples.rb b/test/test_examples.rb index bbe6ddd..3c6f00e 100644..100755 --- a/test/test_examples.rb +++ b/test/test_examples.rb @@ -19,15 +19,15 @@ module ValidationExamples class MajorityIrisCrossvalidation < IrisCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") super end end class LazarIrisCrossvalidation < IrisCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer") super end end @@ -43,15 +43,65 @@ module ValidationExamples class LazarIrisSplit < IrisSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer") super end end class MajorityIrisSplit < IrisSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + ######################################################################################################## + + class EPAFHMSplit < SplitTestValidation + def initialize + @dataset_file = File.new("data/EPAFHM.csv","r") + #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" + @split_ratio = 0.95 + end + end + + class LazarEPAFHMSplit < EPAFHMSplit + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + super + end + end + + class MajorityEPAFHMSplit < EPAFHMSplit + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + ######################################################################################################## + + class EPAFHMCrossvalidation < CrossValidation + def initialize + @dataset_file = File.new("data/EPAFHM.csv","r") + #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" + @num_folds = 10 + end + end + + class MajorityEPAFHMCrossvalidation < EPAFHMCrossvalidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + class LazarEPAFHMCrossvalidation < EPAFHMCrossvalidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end @@ -60,48 +110,85 @@ module ValidationExamples class HamsterSplit < SplitTestValidation def initialize - @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") + @dataset_file = File.new("data/hamster_carcinogenicity.csv","r") + + #@prediction_feature = "http://local-ot/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://local-ot/dataset/1/feature/hamster_carcinogenicity" end end class LazarHamsterSplit < HamsterSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end class MajorityHamsterSplit < HamsterSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end ######################################################################################################## + + class HamsterBootstrapping < BootstrappingValidation + def initialize + #@dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") + @dataset_file = File.new("data/hamster_carcinogenicity.csv","r") + + #@prediction_feature = "http://local-ot/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://local-ot/dataset/1/feature/hamster_carcinogenicity" + end + end + + class LazarHamsterBootstrapping < HamsterBootstrapping + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + super + end + end + + class MajorityHamsterBootstrapping < HamsterBootstrapping + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + ######################################################################################################## class HamsterTrainingTest < TrainingTestValidation def initialize - @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") - @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r") - @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" +# @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") +# @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r") +# @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r") + + @test_target_dataset_file = File.new("data/hamster_carcinogenicity.csv","r") + @training_dataset_file = File.new("data/hamster_carcinogenicity.train.csv","r") + @test_dataset_file = File.new("data/hamster_carcinogenicity.test.csv","r") + + + #@prediction_feature = "http://local-ot/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://local-ot/dataset/1/feature/hamster_carcinogenicity" end end class MajorityHamsterTrainingTest < HamsterTrainingTest def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end class LazarHamsterTrainingTest < HamsterTrainingTest def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end @@ -110,23 +197,26 @@ module ValidationExamples class HamsterCrossvalidation < CrossValidation def initialize - @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") + @dataset_file = File.new("data/hamster_carcinogenicity.csv","r") + + #@prediction_feature = "http://local-ot/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://local-ot/dataset/1/feature/hamster_carcinogenicity" @num_folds = 10 end end class MajorityHamsterCrossvalidation < HamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end class LazarHamsterCrossvalidation < HamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end @@ -142,15 +232,15 @@ module ValidationExamples class MajorityISTHamsterCrossvalidation < ISTHamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end class LazarISTHamsterCrossvalidation < ISTHamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer") super end end @@ -182,6 +272,22 @@ module ValidationExamples ######################################################################################################## + class ISTRatLiverCrossvalidation < CrossValidation + def initialize + @dataset_uri = "http://webservices.in-silico.ch/dataset/26" + @prediction_feature = "http://toxcreate.org/feature#chr_rat_liver_proliferativelesions" + end + end + + class MajorityISTRatLiverCrossvalidation < ISTRatLiverCrossvalidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + ######################################################################################################## + class ISTEpaCrossvalidation < CrossValidation def initialize @dataset_uri = "http://ot-dev.in-silico.ch/dataset/69" @@ -198,12 +304,19 @@ module ValidationExamples end ######################################################################################################## + + + ######################################################################################################## class LR_AmbitCacoModel < ModelValidation def initialize - @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/33" - @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" +# @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/33" +# @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" #@prediction_feature=http://apps.ideaconsult.net:8080/ambit2/feature/22200 + + @model_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/model/33" + @test_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R545" + end end @@ -240,13 +353,183 @@ module ValidationExamples class MajorityCacoTrainingTest < CacoTrainingTest def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") super end end ######################################################################################################## - + + class NtuaModel < ModelValidation + def initialize + @model_uri = "http://opentox.ntua.gr:4000/model/0d8a9a27-3481-4450-bca1-d420a791de9d" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/54" + #@prediction_feature=http://apps.ideaconsult.net:8080/ambit2/feature/22200 + end + end + + ######################################################################################################## + + class TumModel < ModelValidation + def initialize + @model_uri = "http://opentox-dev.informatik.tu-muenchen.de:8080/OpenTox-sec/sec/model/TUMOpenToxModel_M5P_5" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/99488" + end + end + + ######################################################################################################## + + class AmbitModelValidation < ModelValidation + def initialize + @model_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/model/39319" + #@model_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/model/29139" + @test_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401577?max=3" + end + end + + class AmbitBursiModelValidation < ModelValidation + def initialize + @model_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/model/35194" + @test_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401577" + end + end + + class AmbitAquaticModelValidation < ModelValidation + def initialize + @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/130668" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/186293?feature_uris[]=http://apps.ideaconsult.net:8080/ambit2/feature/430904&feature_uris[]=http://apps.ideaconsult.net:8080/ambit2/feature/430905" + @prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/430905" + end + end + + class AmbitTrainingTest < TrainingTestValidation + def initialize + @training_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401560" + #@training_dataset_uri = "http://opentox.informatik.uni-freiburg.de/dataset/317" + @test_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401560" + @prediction_feature = "https://ambit.uni-plovdiv.bg:8443/ambit2/feature/22190" + @algorithm_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/algorithm/LR" + end + end + + class AmbitBursiTrainingTest < TrainingTestValidation + def initialize + @test_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401577" + @training_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401577" + @prediction_feature = "https://ambit.uni-plovdiv.bg:8443/ambit2/feature/26221" + @algorithm_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/algorithm/J48" + end + end + + class AmbitJ48TrainingTest < TrainingTestValidation + def initialize + @test_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/39914" + @training_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/39914" + @prediction_feature = "https://ambit.uni-plovdiv.bg:8443/ambit2/feature/221726" + @algorithm_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/algorithm/J48" + end + end + + class AmbitTrainingTestSplit < SplitTestValidation + def initialize + #@model_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/model/29139" + @dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401560" + #@test_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401560" + @prediction_feature = "https://ambit.uni-plovdiv.bg:8443/ambit2/feature/22190" + @algorithm_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/algorithm/LR" + end + end + + class AmbitBursiTrainingTestSplit < SplitTestValidation + def initialize + @dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401577" + @prediction_feature = "https://ambit.uni-plovdiv.bg:8443/ambit2/feature/26221" + @algorithm_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/algorithm/J48" + end + end + + class AmbitJ48TrainingTestSplit < SplitTestValidation + def initialize + @dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/39914" + @prediction_feature = "https://ambit.uni-plovdiv.bg:8443/ambit2/feature/221726" + @algorithm_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/algorithm/J48" + end + end + + + ######################################################################################################## + + class HamsterTrainingTest < TrainingTestValidation + def initialize +# @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") +# @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r") +# @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r") + + @test_target_dataset_file = File.new("data/hamster_carcinogenicity.csv","r") + @training_dataset_file = File.new("data/hamster_carcinogenicity.train.csv","r") + @test_dataset_file = File.new("data/hamster_carcinogenicity.test.csv","r") + + + #@prediction_feature = "http://local-ot/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://local-ot/dataset/1/feature/hamster_carcinogenicity" + end + end + + class MajorityHamsterTrainingTest < HamsterTrainingTest + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + ######################################################################################################## + + class RepdoseSplit < SplitTestValidation + def initialize + @dataset_file = File.new("data/repdose_classification.csv","r") + end + end + + class LazarRepdoseSplit < RepdoseSplit + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + super + end + end + + class MajorityRepdoseSplit < RepdoseSplit + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + ######################################################################################################## + + class RepdoseCrossValidation < CrossValidation + def initialize + @dataset_file = File.new("data/repdose_classification.csv","r") + end + end + + class LazarRepdoseCrossValidation < RepdoseCrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + super + end + end + + class MajorityRepdoseCrossValidation < RepdoseCrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + ######################################################################################################## + @@list = { "1" => [ LazarHamsterSplit, MajorityHamsterSplit ], "1a" => [ LazarHamsterSplit ], @@ -276,11 +559,48 @@ module ValidationExamples "7a" => [ LazarIrisSplit ], "7b" => [ MajorityIrisSplit ], + "8a" => [ LazarIrisCrossvalidation ], "8b" => [ MajorityIrisCrossvalidation ], "9a" => [ ISTLazarISTIrisCrossvalidation ], "10a" => [ ISTLazarISTEpaCrossvalidation ], + + "11b" => [ MajorityISTRatLiverCrossvalidation ], + + "12" => [ LazarHamsterBootstrapping, MajorityHamsterBootstrapping ], + "12a" => [ LazarHamsterBootstrapping ], + "12b" => [ MajorityHamsterBootstrapping ], + + "13a" => [ LazarEPAFHMSplit ], + "13b" => [ MajorityEPAFHMSplit ], + + "14a" => [ LazarEPAFHMCrossvalidation ], + "14b" => [ MajorityEPAFHMCrossvalidation ], + + "15a" => [ NtuaModel ], + + "16" => [ LazarRepdoseSplit, MajorityRepdoseSplit ], + "16a" => [ LazarRepdoseSplit ], + "16b" => [ MajorityRepdoseSplit ], + + "17" => [ LazarRepdoseCrossValidation, MajorityRepdoseCrossValidation ], + "17a" => [ LazarRepdoseCrossValidation ], + "17b" => [ MajorityRepdoseCrossValidation ], + + "18a" => [ TumModel ], + + "19a" => [ AmbitModelValidation ], + "19b" => [ AmbitTrainingTest ], + "19c" => [ AmbitTrainingTestSplit ], + "19d" => [ AmbitBursiTrainingTest ], + "19e" => [ AmbitBursiModelValidation ], + "19f" => [ AmbitBursiTrainingTestSplit ], + "19g" => [ AmbitJ48TrainingTest ], + "19h" => [ AmbitJ48TrainingTestSplit ], + "19i" => [ AmbitAquaticModelValidation ], + + } def self.list @@ -301,4 +621,4 @@ module ValidationExamples end #puts ValidationExamples.list -#puts ValidationExamples.select("1,2a").inspect
\ No newline at end of file +#puts ValidationExamples.select("1,2a").inspect diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb index 09f7d6c..a5f2867 100644..100755 --- a/test/test_examples_util.rb +++ b/test/test_examples_util.rb @@ -4,21 +4,46 @@ module ValidationExamples class Util @@dataset_uris = {} + @@prediction_features = {} - def self.upload_dataset(file, dataset_service=@@config[:services]["opentox-dataset"], file_type="application/x-yaml") + def self.upload_dataset(file, subjectid=nil, dataset_service=CONFIG[:services]["opentox-dataset"]) #, file_type="application/x-yaml") raise "File not found: "+file.path.to_s unless File.exist?(file.path) if @@dataset_uris[file.path.to_s]==nil - data = File.read(file.path) - data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp - @@dataset_uris[file.path.to_s] = data_uri - LOGGER.debug "uploaded dataset: "+data_uri + LOGGER.debug "uploading file: "+file.path.to_s + if (file.path =~ /yaml$/) + data = File.read(file.path) + #data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp + #@@dataset_uris[file.path.to_s] = data_uri + #LOGGER.debug "uploaded dataset: "+data_uri + d = OpenTox::Dataset.create(CONFIG[:services]["opentox-dataset"], subjectid) + d.load_yaml(data) + d.save( subjectid ) + @@dataset_uris[file.path.to_s] = d.uri + elsif (file.path =~ /csv$/) + d = OpenTox::Dataset.create_from_csv_file(file.path, subjectid) + raise "num features not 1 (="+d.features.keys.size.to_s+"), what to predict??" if d.features.keys.size != 1 + @@prediction_features[file.path.to_s] = d.features.keys[0] + @@dataset_uris[file.path.to_s] = d.uri + elsif (file.path =~ /rdf$/) + d = OpenTox::Dataset.create(CONFIG[:services]["opentox-dataset"], subjectid) + d.load_rdfxml_file(file, subjectid) + d.save(subjectid) + @@dataset_uris[file.path.to_s] = d.uri + else + raise "unknown file type: "+file.path.to_s + end + LOGGER.debug "uploaded dataset: "+d.uri else LOGGER.debug "file already uploaded: "+@@dataset_uris[file.path.to_s] end return @@dataset_uris[file.path.to_s] end - def self.build_compare_report(validation_examples) + def self.prediction_feature_for_file(file) + @@prediction_features[file.path.to_s] + end + + def self.build_compare_report(validation_examples, subjectid) @comp = validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri return nil if @comp == :model_uri @@ -27,39 +52,55 @@ module ValidationExamples to_compare << v.validation_uri if v.validation_uri and v.validation_error==nil end return nil if to_compare.size < 2 - begin - return validation_post "report/algorithm_comparison",{ :validation_uris => to_compare.join("\n") } - rescue => ex - return "error creating comparison report "+ex.message - end + #begin + return validation_post "report/algorithm_comparison",{ :validation_uris => to_compare.join("\n") }, subjectid + #rescue => ex + #return "error creating comparison report "+ex.message + #end end - def self.validation_post(uri, params) + def self.validation_post(uri, params, subjectid, waiting_task=nil ) + + params[:subjectid] = subjectid if subjectid if $test_case - #puts "posting: "+uri+","+params.inspect - $test_case.post uri,params + $test_case.post uri,params return wait($test_case.last_response.body) else - return OpenTox::RestClientWrapper.post(File.join(@@config[:services]["opentox-validation"],uri),params) + return OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],uri),params,nil,waiting_task).to_s end end - def self.validation_get(uri, accept_header='application/rdf+xml') + def self.validation_get(uri, subjectid, accept_header='application/rdf+xml') + params = {} + params[:subjectid] = subjectid if subjectid if $test_case #puts "getting "+uri+","+accept_header - $test_case.get uri,nil,'HTTP_ACCEPT' => accept_header + $test_case.get uri,params,'HTTP_ACCEPT' => accept_header + return wait($test_case.last_response.body) + else + params[:accept] = accept_header + return OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"],uri),params) + end + end + + def self.validation_delete(uri, accept_header='application/rdf+xml') + + if $test_case + $test_case.delete uri,{:subjectid => SUBJECTID},'HTTP_ACCEPT' => accept_header return wait($test_case.last_response.body) else - return OpenTox::RestClientWrapper.get(File.join(@@config[:services]["opentox-validation"],uri),{:accept => accept_header}) + return OpenTox::RestClientWrapper.delete(File.join(CONFIG[:services]["opentox-validation"],uri),{:accept => accept_header,:subjectid => SUBJECTID}) end end + def self.wait(uri) - if OpenTox::Utils.task_uri?(uri) - task = OpenTox::Task.find(uri) + if uri.task_uri? + task = OpenTox::Task.find(uri.to_s.chomp) task.wait_for_completion - raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error? - uri = task.resultURI + #raise "task failed: "+uri.to_s+", description: '"+task.description.to_s+"'" if task.error? + LOGGER.error "task failed :\n"+task.to_yaml if task.error? + uri = task.result_uri end uri end @@ -67,14 +108,14 @@ module ValidationExamples def self.verify_crossvalidation(val_yaml) val = YAML.load(val_yaml) - puts val.inspect + #puts val.inspect assert_integer val["random_seed".to_sym],nil,nil,"random_seed" assert_boolean val["stratified".to_sym],"stratified" assert_integer val["num_folds".to_sym],0,1000,"num_folds" num_folds = val["num_folds".to_sym].to_i - validations = val["validations".to_sym] + validations = val["validation_uris".to_sym] assert_int_equal(num_folds, validations.size, "num_folds != validations.size") end @@ -82,7 +123,7 @@ module ValidationExamples val = YAML.load(val_yaml) - puts val.inspect + #puts val.inspect assert_integer val["num_instances".to_sym],0,1000,"num_instances" num_instances = val["num_instances".to_sym].to_i @@ -103,7 +144,7 @@ module ValidationExamples class_value_stats.each do |cvs| class_values << cvs["class_value".to_sym] end - puts class_values.inspect + #puts class_values.inspect confusion_matrix = class_stats["confusion_matrix".to_sym] confusion_matrix_cells = confusion_matrix["confusion_matrix_cell".to_sym] @@ -118,12 +159,40 @@ module ValidationExamples end end + def self.compare_yaml_and_owl(hash, owl, nested_params=[] ) + + hash.each do |k,v| + p = nested_params + [ k.to_s.to_rdf_format ] + if (v.is_a?(Hash)) + compare_yaml_and_owl( v, owl, p ) + elsif (v.is_a?(Array)) + v.each do |vv| + compare_yaml_and_owl( vv, owl, p ) + end + else + owl_value = owl.get_nested( p ) + if owl_value.size == 0 + raise "owl_value is nil, yaml value is '"+v.to_s+"'" unless v==nil or v.to_s.size==0 + elsif owl_value.size == 1 + assert_equal(v, owl_value[0], p.join(".")+" (yaml != rdf)") + else + raise p.join(".")+" yaml value '"+v.to_s+"' not included in rdf values '"+ + owl_value.inspect+"'" unless owl_value.include?(v) + end + end + end + end + private def self.assert_not_nil(val,msg_suffix=nil) raise msg_suffix.to_s+" is nil" if val==nil end def self.assert_int_equal(val1,val2,msg_suffix=nil) + assert_equal(val1, val2, msg_suffix) + end + + def self.assert_equal(val1,val2,msg_suffix=nil) raise msg_suffix.to_s+" not equal: "+val1.to_s+" != "+val2.to_s unless val1==val2 end @@ -168,13 +237,14 @@ module ValidationExamples :split_ratio, :random_seed, :num_folds, - :stratified + :stratified, + :subjectid #results attr_accessor :validation_uri, :report_uri, :validation_error, :report_error - + def upload_files [[:test_dataset_uri, :test_dataset_file], [:test_target_dataset_uri, :test_target_dataset_file], @@ -183,9 +253,16 @@ module ValidationExamples uri = a[0] file = a[1] if send(uri)==nil and send(file)!=nil - send("#{uri.to_s}=".to_sym, Util.upload_dataset(send(file))) + dataset_uri = Util.upload_dataset(send(file), @subjectid) + send("#{uri.to_s}=".to_sym, dataset_uri) + @uploaded_datasets = [] unless @uploaded_datasets + @uploaded_datasets << dataset_uri end end + + if (params.include?(:prediction_feature) and @prediction_feature==nil and @dataset_uri and @dataset_file) + @prediction_feature = Util.prediction_feature_for_file(@dataset_file) + end end def check_requirements @@ -194,30 +271,78 @@ module ValidationExamples end end - def report - begin - @report_uri = Util.validation_post '/report/'+report_type,{:validation_uris => @validation_uri} - rescue => ex - @report_error = ex.message + def delete + #begin + if @validation_uri =~ /crossvalidation/ + cv = "crossvalidation/" + else + cv = "" + end + Util.validation_delete '/'+cv+@validation_uri.split('/')[-1] if @validation_uri + #rescue => ex + #puts "Could not delete validation: "+ex.message + #end + #begin + Util.validation_delete '/report/'+report_type+'/'+@report_uri.split('/')[-1] if @report_uri + #rescue => ex + #puts "Could not delete report:' "+@report_uri+" "+ex.message + #end + @uploaded_datasets.each do |d| + # begin + puts "deleting dataset "+d + OpenTox::RestClientWrapper.delete(d,{:subjectid => SUBJECTID}) +# rescue => ex + #puts "Could not delete dataset:' "+d+" "+ex.message + #end end end - def validate - begin - @validation_uri = Util.validation_post '/'+validation_type, get_params - rescue => ex - @validation_error = ex.message - LOGGER.error ex.message + def report( waiting_task=nil ) + #begin + @report_uri = Util.validation_post '/report/'+report_type,{:validation_uris => @validation_uri},@subjectid,waiting_task if @validation_uri + Util.validation_get "/report/"+report_uri.split("/")[-2]+"/"+report_uri.split("/")[-1], @subjectid if @report_uri + #rescue => ex + #puts "could not create report: "+ex.message + #raise ex + #@report_error = ex.message + #end + end + + def validate( waiting_task=nil ) + #begin + @validation_uri = Util.validation_post '/'+validation_type, get_params, @subjectid, waiting_task + #rescue => ex + #puts "could not validate: "+ex.message + #@validation_error = ex.message + #LOGGER.error ex.message + #end + end + + def compare_yaml_vs_rdf + if @validation_uri + yaml = YAML.load(Util.validation_get(@validation_uri.split("/")[-1],@subjectid,'application/x-yaml')) + owl = OpenTox::Owl.from_data(Util.validation_get(@validation_uri.split("/")[-1],@subjectid),@validation_uri,"Validation") + Util.compare_yaml_and_owl(yaml,owl) + end + if @report_uri + yaml = YAML.load(Util.validation_get(@report_uri.split("/")[-3..-1].join("/"),@subjectid,'application/x-yaml')) + owl = OpenTox::Owl.from_data(Util.validation_get(@report_uri.split("/")[-3..-1].join("/"),@subjectid),@report_uri,"ValidationReport") + Util.compare_yaml_and_owl(yaml,owl) + Util.validation_get(@report_uri.split("/")[-3..-1].join("/"),@subjectid,'text/html') + else + puts "no report" end end + def verify_yaml + raise "cannot very validation, validation_uri is null" unless @validation_uri if @validation_uri =~ /crossvalidation/ Util.verify_crossvalidation(Util.validation_get("crossvalidation/"+@validation_uri.split("/")[-1],'application/x-yaml')) Util.validation_get("crossvalidation/"+@validation_uri.split("/")[-1]+"/statistics",'application/x-yaml') Util.verify_validation(Util.validation_get("crossvalidation/"+@validation_uri.split("/")[-1]+"/statistics",'application/x-yaml')) else - Util.verify_validation(Util.validation_get(@validation_uri.split("/")[-1],'application/x-yaml')) + Util.verify_validation(Util.validation_get(@validation_uri.split("/")[-1],@subjectid,'application/x-yaml')) end end @@ -252,6 +377,10 @@ module ValidationExamples def opt_params [ :prediction_feature, :test_target_dataset_uri ] end + + def validation_type + "test_set_validation" + end end class TrainingTestValidation < ValidationExample @@ -262,6 +391,10 @@ module ValidationExamples def opt_params [ :algorithm_params, :test_target_dataset_uri ] end + + def validation_type + "training_test_validation" + end end class SplitTestValidation < ValidationExample @@ -278,6 +411,20 @@ module ValidationExamples end end + class BootstrappingValidation < ValidationExample + def params + [:algorithm_uri, :dataset_uri, :prediction_feature] + end + + def opt_params + [ :algorithm_params, :random_seed ] + end + + def validation_type + "bootstrapping" + end + end + class CrossValidation < ValidationExample def params [:algorithm_uri, :dataset_uri, :prediction_feature] @@ -295,4 +442,4 @@ module ValidationExamples "crossvalidation" end end -end
\ No newline at end of file +end diff --git a/test/unit_test.rb b/test/unit_test.rb new file mode 100644 index 0000000..11b0759 --- /dev/null +++ b/test/unit_test.rb @@ -0,0 +1,189 @@ +#TEST_USER = "mgtest" +#TEST_PW = "mgpasswd" +#ENV['RACK_ENV'] = 'test' + +require "rubygems" +require "sinatra" +require "uri" +require "yaml" +require 'application.rb' +require 'test/unit' +require 'rack/test' +require 'lib/test_util.rb' +require 'test/test_examples.rb' + +TEST_USER = "guest" +TEST_PW = "guest" + +#LOGGER = OTLogger.new(STDOUT) +#LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S " +#LOGGER.formatter = Logger::Formatter.new + +module Sinatra + set :raise_errors, false + set :show_exceptions, false +end + +class Exception + def message + errorCause ? errorCause.to_yaml : to_s + end +end + +class ValidationTest < Test::Unit::TestCase + include Rack::Test::Methods + include Lib::TestUtil + + def global_setup + puts "login and upload datasets" + if AA_SERVER + @@subjectid = OpenTox::Authorization.authenticate(TEST_USER,TEST_PW) + raise "could not log in" unless @@subjectid + puts "logged in: "+@@subjectid.to_s + else + puts "AA disabled" + @@subjectid = nil + end + f = File.new("data/hamster_carcinogenicity.mini.csv") + @@data_class_mini = ValidationExamples::Util.upload_dataset(f, @@subjectid) + @@feat_class_mini = ValidationExamples::Util.prediction_feature_for_file(f) + end + + def global_teardown + puts "delete and logout" + OpenTox::Dataset.find(@@data_class_mini,@@subjectid).delete(@@subjectid) if defined?@@data_class_mini + @@cv.delete(@@subjectid) if defined?@@cv + @@report.delete(@@subjectid) if defined?@@report + @@qmrfReport.delete(@@subjectid) if defined?@@qmrfReport + OpenTox::Authorization.logout(@@subjectid) if AA_SERVER + end + + def test_crossvalidation + puts "test_crossvalidation" + #assert_rest_call_error OpenTox::NotFoundError do + # OpenTox::Crossvalidation.find(File.join(CONFIG[:services]["opentox-validation"],"crossvalidation/noexistingid")) + #end + p = { + :dataset_uri => @@data_class_mini, + :algorithm_uri => File.join(CONFIG[:services]["opentox-algorithm"],"lazar"), + :algorithm_params => "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc"), + :prediction_feature => @@feat_class_mini, + :num_folds => 2 } + t = OpenTox::SubTask.new(nil,0,1) + def t.progress(pct) + if !defined?@last_msg or @last_msg+3<Time.new + puts "waiting for crossvalidation: "+pct.to_s + @last_msg=Time.new + end + end + def t.waiting_for(task_uri); end + cv = OpenTox::Crossvalidation.create(p, @@subjectid, t) + assert cv.uri.uri? + if @@subjectid + assert_rest_call_error OpenTox::NotAuthorizedError do + OpenTox::Crossvalidation.find(cv.uri) + end + end + cv = OpenTox::Crossvalidation.find(cv.uri, @@subjectid) + assert cv.uri.uri? + if @@subjectid + assert_rest_call_error OpenTox::NotAuthorizedError do + cv.summary(cv) + end + end + summary = cv.summary(@@subjectid) + assert_kind_of Hash,summary + @@cv = cv + end + + def test_crossvalidation_report + #@@cv = OpenTox::Crossvalidation.find("http://local-ot/validation/crossvalidation/48", @@subjectid) + + puts "test_crossvalidation_report" + assert defined?@@cv,"no crossvalidation defined" + assert_kind_of OpenTox::Crossvalidation,@@cv + #assert_rest_call_error OpenTox::NotFoundError do + # OpenTox::CrossvalidationReport.find_for_crossvalidation(@@cv.uri) + #end + if @@subjectid + assert_rest_call_error OpenTox::NotAuthorizedError do + OpenTox::CrossvalidationReport.create(@@cv.uri) + end + end + report = OpenTox::CrossvalidationReport.create(@@cv.uri,@@subjectid) + assert report.uri.uri? + if @@subjectid + assert_rest_call_error OpenTox::NotAuthorizedError do + OpenTox::CrossvalidationReport.find(report.uri) + end + end + report = OpenTox::CrossvalidationReport.find(report.uri,@@subjectid) + assert report.uri.uri? + report2 = OpenTox::CrossvalidationReport.find_for_crossvalidation(@@cv.uri,@@subjectid) + assert_equal report.uri,report2.uri + report3_uri = @@cv.find_or_create_report(@@subjectid) + assert_equal report.uri,report3_uri + @report = report2 + end + + def test_qmrf_report + #@@cv = OpenTox::Crossvalidation.find("http://local-ot/validation/crossvalidation/47", @@subjectid) + + puts "test_qmrf_report" + assert defined?@@cv,"no crossvalidation defined" + + validations = @@cv.metadata[OT.validation] + assert_kind_of Array,validations + assert validations.size==@@cv.metadata[OT.numFolds] + + val = OpenTox::Validation.find(validations[0], @@subjectid) + model_uri = val.metadata[OT.model] + model = OpenTox::Model::Generic.find(model_uri, @@subjectid) + assert model!=nil + + #assert_rest_call_error OpenTox::NotFoundError do + # OpenTox::QMRFReport.find_for_model(model_uri, @@subjectid) + #end + + @@qmrfReport = OpenTox::QMRFReport.create(model_uri, @@subjectid) + end + + ################### utils and overrides ########################## + + def app + Sinatra::Application + end + + # checks RestCallError type + def assert_rest_call_error( ex ) + if ex==OpenTox::NotAuthorizedError and @@subjectid==nil + puts "AA disabled: skipping test for not authorized" + return + end + begin + yield + rescue OpenTox::RestCallError => e + report = e.errorCause + while report.errorCause + report = report.errorCause + end + assert_equal report.errorType,ex.to_s + end + end + + # hack to have a global_setup and global_teardown + def teardown + if((@@expected_test_count-=1) == 0) + global_teardown + end + end + def setup + unless defined?@@expected_test_count + @@expected_test_count = (self.class.instance_methods.reject{|method| method[0..3] != 'test'}).length + global_setup + end + end + +end + + |