diff options
author | mguetlein <martin.guetlein@gmail.com> | 2011-01-13 15:18:45 +0100 |
---|---|---|
committer | mguetlein <martin.guetlein@gmail.com> | 2011-01-13 15:18:45 +0100 |
commit | eeb0b6df2dbdae29ccf3f4ccd923002e0ed65506 (patch) | |
tree | c0b0956d5b107e613c3236ede71462fa7f18cf05 /test | |
parent | 5786ebd5095451e928d9811bd6bbd98af72de99d (diff) |
adjust to new wrapper, replace activerecord with datamapper (remove activerecord code), is_classification to feature_type
Diffstat (limited to 'test')
-rwxr-xr-x[-rw-r--r--] | test/test_application.rb | 3 | ||||
-rwxr-xr-x[-rw-r--r--] | test/test_examples.rb | 114 | ||||
-rwxr-xr-x[-rw-r--r--] | test/test_examples_util.rb | 96 |
3 files changed, 167 insertions, 46 deletions
diff --git a/test/test_application.rb b/test/test_application.rb index 48cced7..c6c7963 100644..100755 --- a/test/test_application.rb +++ b/test/test_application.rb @@ -2,7 +2,7 @@ post '/test_validation/?' do validationExamples = ValidationExamples.select(params[:select]) return "please \"select\" a single validation example:\n"+ValidationExamples.list if validationExamples.size!=1 or validationExamples[0].size!=1 - OpenTox::Task.as_task("Test validation",url_for("/test_validation",:full), params) do + task = OpenTox::Task.create("Test validation",url_for("/test_validation",:full)) do #,params v = validationExamples[0][0] ex = v.new ex.upload_files @@ -13,5 +13,6 @@ post '/test_validation/?' do raise ex.report_error if ex.report_error ex.validation_uri + (params[:report] ? ","+ex.report_uri : "") end + task.uri end diff --git a/test/test_examples.rb b/test/test_examples.rb index 8927fb1..63fb505 100644..100755 --- a/test/test_examples.rb +++ b/test/test_examples.rb @@ -19,15 +19,15 @@ module ValidationExamples class MajorityIrisCrossvalidation < IrisCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") super end end class LazarIrisCrossvalidation < IrisCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer") super end end @@ -43,15 +43,65 @@ module ValidationExamples class LazarIrisSplit < IrisSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer") super end end class MajorityIrisSplit < IrisSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + ######################################################################################################## + + class EPAFHMSplit < SplitTestValidation + def initialize + @dataset_file = File.new("data/EPAFHM.csv","r") + #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" + @split_ratio = 0.95 + end + end + + class LazarEPAFHMSplit < EPAFHMSplit + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + super + end + end + + class MajorityEPAFHMSplit < EPAFHMSplit + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + ######################################################################################################## + + class EPAFHMCrossvalidation < CrossValidation + def initialize + @dataset_file = File.new("data/EPAFHM.csv","r") + #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" + @num_folds = 10 + end + end + + class MajorityEPAFHMCrossvalidation < EPAFHMCrossvalidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + class LazarEPAFHMCrossvalidation < EPAFHMCrossvalidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end @@ -61,21 +111,22 @@ module ValidationExamples class HamsterSplit < SplitTestValidation def initialize @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity" end end class LazarHamsterSplit < HamsterSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end class MajorityHamsterSplit < HamsterSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end @@ -85,21 +136,22 @@ module ValidationExamples class HamsterBootstrapping < BootstrappingValidation def initialize @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity" end end class LazarHamsterBootstrapping < HamsterBootstrapping def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end class MajorityHamsterBootstrapping < HamsterBootstrapping def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end @@ -111,21 +163,22 @@ module ValidationExamples @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r") @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity" end end class MajorityHamsterTrainingTest < HamsterTrainingTest def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end class LazarHamsterTrainingTest < HamsterTrainingTest def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end @@ -135,22 +188,23 @@ module ValidationExamples class HamsterCrossvalidation < CrossValidation def initialize @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity" @num_folds = 10 end end class MajorityHamsterCrossvalidation < HamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end class LazarHamsterCrossvalidation < HamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end @@ -166,15 +220,15 @@ module ValidationExamples class MajorityISTHamsterCrossvalidation < ISTHamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end class LazarISTHamsterCrossvalidation < ISTHamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer") super end end @@ -215,7 +269,7 @@ module ValidationExamples class MajorityISTRatLiverCrossvalidation < ISTRatLiverCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end @@ -283,7 +337,7 @@ module ValidationExamples class MajorityCacoTrainingTest < CacoTrainingTest def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") super end end @@ -331,6 +385,12 @@ module ValidationExamples "12" => [ LazarHamsterBootstrapping, MajorityHamsterBootstrapping ], "12a" => [ LazarHamsterBootstrapping ], "12b" => [ MajorityHamsterBootstrapping ], + + "13a" => [ LazarEPAFHMSplit ], + "13b" => [ MajorityEPAFHMSplit ], + + "14a" => [ LazarEPAFHMCrossvalidation ], + "14b" => [ MajorityEPAFHMCrossvalidation ], } def self.list diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb index cc31fe5..b9a1c99 100644..100755 --- a/test/test_examples_util.rb +++ b/test/test_examples_util.rb @@ -4,20 +4,40 @@ module ValidationExamples class Util @@dataset_uris = {} + @@prediction_features = {} - def self.upload_dataset(file, dataset_service=@@config[:services]["opentox-dataset"], file_type="application/x-yaml") + def self.upload_dataset(file, dataset_service=CONFIG[:services]["opentox-dataset"]) #, file_type="application/x-yaml") raise "File not found: "+file.path.to_s unless File.exist?(file.path) if @@dataset_uris[file.path.to_s]==nil - data = File.read(file.path) - data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp - @@dataset_uris[file.path.to_s] = data_uri - LOGGER.debug "uploaded dataset: "+data_uri + LOGGER.debug "uploading file: "+file.path.to_s + if (file.path =~ /yaml$/) + data = File.read(file.path) + #data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp + #@@dataset_uris[file.path.to_s] = data_uri + #LOGGER.debug "uploaded dataset: "+data_uri + d = OpenTox::Dataset.create + d.load_yaml(data) + d.save + @@dataset_uris[file.path.to_s] = d.uri + elsif (file.path =~ /csv$/) + d = OpenTox::Dataset.create_from_csv_file(file.path) + raise "num features not 1 (="+d.features.keys.size.to_s+"), what to predict??" if d.features.keys.size != 1 + @@prediction_features[file.path.to_s] = d.features.keys[0] + @@dataset_uris[file.path.to_s] = d.uri + else + raise "unknown file type: "+file.path.to_s + end + LOGGER.debug "uploaded dataset: "+d.uri else LOGGER.debug "file already uploaded: "+@@dataset_uris[file.path.to_s] end return @@dataset_uris[file.path.to_s] end + def self.prediction_feature_for_file(file) + @@prediction_features[file.path.to_s] + end + def self.build_compare_report(validation_examples) @comp = validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri @@ -35,12 +55,12 @@ module ValidationExamples end def self.validation_post(uri, params) + if $test_case - #puts "posting: "+uri+","+params.inspect - $test_case.post uri,params + $test_case.post uri,params return wait($test_case.last_response.body) else - return OpenTox::RestClientWrapper.post(File.join(@@config[:services]["opentox-validation"],uri),params).to_s + return OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],uri),params).to_s end end @@ -50,16 +70,27 @@ module ValidationExamples $test_case.get uri,nil,'HTTP_ACCEPT' => accept_header return wait($test_case.last_response.body) else - return OpenTox::RestClientWrapper.get(File.join(@@config[:services]["opentox-validation"],uri),{:accept => accept_header}) + return OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"],uri),{:accept => accept_header}) + end + end + + def self.validation_delete(uri, accept_header='application/rdf+xml') + + if $test_case + $test_case.delete uri,nil,'HTTP_ACCEPT' => accept_header + return wait($test_case.last_response.body) + else + return OpenTox::RestClientWrapper.delete(File.join(CONFIG[:services]["opentox-validation"],uri),{:accept => accept_header}) end end + def self.wait(uri) - if OpenTox::Utils.task_uri?(uri) - task = OpenTox::Task.find(uri) + if uri.task_uri? + task = OpenTox::Task.find(uri.to_s.chomp) task.wait_for_completion - raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error? - uri = task.resultURI + raise "task failed: "+uri.to_s+", error is:\n"+task.description.to_s if task.error? + uri = task.result_uri end uri end @@ -214,6 +245,10 @@ module ValidationExamples send("#{uri.to_s}=".to_sym, Util.upload_dataset(send(file))) end end + + if (params.include?(:prediction_feature) and @prediction_feature==nil and @dataset_uri and @dataset_file) + @prediction_feature = Util.prediction_feature_for_file(@dataset_file) + end end def check_requirements @@ -222,21 +257,42 @@ module ValidationExamples end end + def delete + begin + if @validation_uri =~ /crossvalidation/ + cv = "crossvalidation/" + else + cv = "" + end + Util.validation_delete '/'+cv+@validation_uri.split('/')[-1] if @validation_uri + rescue => ex + puts "Could not delete validation: "+ex.message + end + begin + Util.validation_delete '/report/'+report_type+'/'+@report_uri.split('/')[-1] if @report_uri + rescue => ex + puts "Could not delete report:' "+@report_uri+" "+ex.message + end + end + def report begin @report_uri = Util.validation_post '/report/'+report_type,{:validation_uris => @validation_uri} rescue => ex + puts "could not create report: "+ex.message + raise ex @report_error = ex.message end end def validate - #begin + begin @validation_uri = Util.validation_post '/'+validation_type, get_params - #rescue => ex - # @validation_error = ex.message - # LOGGER.error ex.message - #end + rescue => ex + puts "could not validate: "+ex.message + @validation_error = ex.message + LOGGER.error ex.message + end end def compare_yaml_vs_rdf @@ -308,6 +364,10 @@ module ValidationExamples def opt_params [ :algorithm_params, :test_target_dataset_uri ] end + + def validation_type + "training_test_validation" + end end class SplitTestValidation < ValidationExample |