diff options
author | mguetlein <martin.guetlein@gmail.com> | 2010-05-31 17:25:21 +0200 |
---|---|---|
committer | mguetlein <martin.guetlein@gmail.com> | 2010-05-31 17:25:21 +0200 |
commit | 7ce7b16b6b0d2e965e92ae28e6f58644203c1cab (patch) | |
tree | a31f523060c15fb6a9525b010959eda3330f0b07 /test | |
parent | d09a2f15a7dc499545732d7b8b45601533252ed9 (diff) |
test/nightly rewrite
Diffstat (limited to 'test')
-rw-r--r-- | test/test_application.rb | 17 | ||||
-rw-r--r-- | test/test_examples.rb | 214 | ||||
-rw-r--r-- | test/test_examples_util.rb | 194 |
3 files changed, 425 insertions, 0 deletions
diff --git a/test/test_application.rb b/test/test_application.rb new file mode 100644 index 0000000..cbc1057 --- /dev/null +++ b/test/test_application.rb @@ -0,0 +1,17 @@ + +post '/test_validation/?' do + validationExamples = ValidationExamples.select(params[:select]) + return "please \"select\" a single validation example:\n"+ValidationExamples.list if validationExamples.size!=1 or validationExamples[0].size!=1 + OpenTox::Task.as_task do + v = validationExamples[0][0] + ex = v.new + ex.upload_files + ex.check_requirements + ex.validate + raise ex.validation_error if ex.validation_error + ex.report if params[:report] + raise ex.report_error if ex.report_error + ex.validation_uri + (params[:report] ? ","+ex.report_uri : "") + end +end + diff --git a/test/test_examples.rb b/test/test_examples.rb new file mode 100644 index 0000000..d08e358 --- /dev/null +++ b/test/test_examples.rb @@ -0,0 +1,214 @@ + +require 'test/test_examples_util.rb' + +class Class + def humanize + self.to_s.gsub(/.*::/, "").gsub(/([^^A-Z_])([A-Z])/, '\1-\2').gsub(/_/,"-") + end +end + +module ValidationExamples + + ######################################################################################################## + + class ABSTRACTHamsterSplit < SplitTestValidation + def initialize + @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") + @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + end + end + + class LazarHamsterSplit < ABSTRACTHamsterSplit + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + class MajorityHamsterSplit < ABSTRACTHamsterSplit + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + ######################################################################################################## + + class ABSTRACTHamsterTrainingTest < TrainingTestValidation + def initialize + @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") + @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r") + @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r") + @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + end + end + + class MajorityHamsterTrainingTest < ABSTRACTHamsterTrainingTest + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + class LazarHamsterTrainingTest < ABSTRACTHamsterTrainingTest + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + ######################################################################################################## + + class ABSTRACTHamsterCrossvalidation < CrossValidation + def initialize + @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") + @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + end + end + + class MajorityHamsterCrossvalidation < ABSTRACTHamsterCrossvalidation + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + class LazarHamsterCrossvalidation < ABSTRACTHamsterCrossvalidation + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + ######################################################################################################## + + class ABSTRACT_ISTHamsterCrossvalidationInsilico < CrossValidation + def initialize + @dataset_uri = "http://webservices.in-silico.ch/dataset/108" + @prediction_feature = "http://toxcreate.org/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + end + end + + class MajorityISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + class LazarISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + class ISTLazarISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico + def initialize + @algorithm_uri = "http://webservices.in-silico.ch/algorithm/lazar" + @algorithm_params = "feature_generation_uri=http://webservices.in-silico.ch/algorithm/fminer" + super + end + end + + + ######################################################################################################## + + class LR_AmbitCacoModel < ModelValidation + def initialize + @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/33" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + #@prediction_feature=http://apps.ideaconsult.net:8080/ambit2/feature/22200 + end + end + + ######################################################################################################## + + class ABSTRACTCacoTrainingTest < TrainingTestValidation + def initialize + @training_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798" + @test_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353" + @prediction_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510" + end + end + + class LR_AmbitCacoTrainingTest < ABSTRACTCacoTrainingTest + def initialize + @algorithm_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/LR" + super + end + end + + class MLR_NTUA_CacoTrainingTest < ABSTRACTCacoTrainingTest + def initialize + @algorithm_uri = "http://opentox.ntua.gr:3003/algorithm/mlr" + super + end + end + + class MLR_NTUA2_CacoTrainingTest < ABSTRACTCacoTrainingTest + def initialize + @algorithm_uri = "http://opentox.ntua.gr:3004/algorithm/mlr" + super + end + end + + class MajorityCacoTrainingTest < ABSTRACTCacoTrainingTest + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + ######################################################################################################## + + @@list = { + "1" => [ LazarHamsterSplit, MajorityHamsterSplit ], + "1a" => [ LazarHamsterSplit ], + "1b" => [ MajorityHamsterSplit ], + + "2" => [ LazarHamsterTrainingTest, MajorityHamsterTrainingTest ], + "2a" => [ LazarHamsterTrainingTest ], + "2b" => [ MajorityHamsterTrainingTest ], + + "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation ], + "3a" => [ LazarHamsterCrossvalidation ], + "3b" => [ MajorityHamsterCrossvalidation ], + + "4" => [ MajorityISTHamsterCrossvalidation, LazarISTHamsterCrossvalidation, ISTLazarISTHamsterCrossvalidation ], + "4a" => [ MajorityISTHamsterCrossvalidation ], + "4b" => [ LazarISTHamsterCrossvalidation ], + "4c" => [ ISTLazarISTHamsterCrossvalidation ], + + "5a" => [ LR_AmbitCacoModel ], + + "6" => [ LR_AmbitCacoTrainingTest, MLR_NTUA_CacoTrainingTest, MLR_NTUA2_CacoTrainingTest, MajorityCacoTrainingTest ], + "6a" => [ LR_AmbitCacoTrainingTest ], + "6b" => [ MLR_NTUA_CacoTrainingTest ], + "6c" => [ MLR_NTUA2_CacoTrainingTest ], + "6d" => [ MajorityCacoTrainingTest ], + } + + def self.list + @@list.sort.collect{|k,v| k+":\t"+v.collect{|vv| vv.humanize}.join("\n\t")+"\n"} #.join("\n") + end + + def self.select(csv_keys) + res = [] + if csv_keys!=nil and csv_keys.size>0 + csv_keys.split(",").each do |k| + raise "no key "+k.to_s unless @@list.has_key?(k) + res << @@list[k] + end + end + return res + end + +end + +#puts ValidationExamples.list +#puts ValidationExamples.select("1,2a").inspect
\ No newline at end of file diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb new file mode 100644 index 0000000..49e0336 --- /dev/null +++ b/test/test_examples_util.rb @@ -0,0 +1,194 @@ + +module ValidationExamples + + class Util + + @@dataset_uris = {} + + def self.upload_dataset(file, dataset_service=@@config[:services]["opentox-dataset"], file_type="text/x-yaml") + raise "File not found: "+file.path.to_s unless File.exist?(file.path) + if @@dataset_uris[file.path.to_s]==nil + data = File.read(file.path) + data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp + @@dataset_uris[file.path.to_s] = data_uri + LOGGER.debug "uploaded dataset: "+data_uri + else + LOGGER.debug "file already uploaded: "+@@dataset_uris[file.path.to_s] + end + return @@dataset_uris[file.path.to_s] + end + + def self.build_compare_report(validation_examples) + + @comp = validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri + return nil if @comp == :model_uri + to_compare = [] + validation_examples.each do |v| + to_compare << v.validation_uri if v.validation_uri and v.validation_error==nil + end + return nil if to_compare.size < 2 + begin + return validation_post "report/algorithm_comparison",{ :validation_uris => to_compare.join("\n") } + rescue => ex + return "error creating comparison report "+ex.message + end + end + + def self.validation_post(uri, params) + if $test_case + #puts "posting: "+uri+","+params.inspect + $test_case.post uri,params + return wait($test_case.last_response.body) + else + return OpenTox::RestClientWrapper.post(File.join(@@config[:services]["opentox-validation"],uri),params) + end + end + + def self.wait(uri) + if OpenTox::Utils.task_uri?(uri) + task = OpenTox::Task.find(uri) + task.wait_for_completion + raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error? + uri = task.resultURI + end + uri + end + + end + + class ValidationExample + + #params + attr_accessor :name, + :prediction_feature, + :algorithm_uri, + :model_uri, + :test_dataset_uri, + :test_dataset_file, + :test_target_dataset_uri, + :test_target_dataset_file, + :training_dataset_uri, + :training_dataset_file, + :dataset_uri, + :dataset_file, + :algorithm_params, + :split_ratio, + :random_seed, + :num_folds, + :stratified + #results + attr_accessor :validation_uri, + :report_uri, + :validation_error, + :report_error + + def upload_files + [[:test_dataset_uri, :test_dataset_file], + [:test_target_dataset_uri, :test_target_dataset_file], + [:training_dataset_uri, :training_dataset_file], + [:dataset_uri, :dataset_file]].each do |a| + uri = a[0] + file = a[1] + if send(uri)==nil and send(file)!=nil + send("#{uri.to_s}=".to_sym, Util.upload_dataset(send(file))) + end + end + end + + def check_requirements + params.each do |r| + raise "values not set: "+r.to_s if send(r)==nil + end + end + + def report + begin + @report_uri = Util.validation_post '/report/'+report_type,{:validation_uris => @validation_uri} + rescue => ex + @report_error = ex.message + end + end + + def validate + begin + @validation_uri = Util.validation_post '/'+validation_type, get_params + rescue => ex + @validation_error = ex.message + end + end + + def title + self.class.humanize + end + + protected + def report_type + "validation" + end + + def validation_type + "" + end + + def get_params + p = {} + ( params + opt_params ).each do |pp| + p[pp] = send(pp) if send(pp)!=nil + end + return p + end + end + + class ModelValidation < ValidationExample + + def params + [:model_uri, :test_dataset_uri] + end + + def opt_params + [ :prediction_feature, :test_target_dataset_uri ] + end + end + + class TrainingTestValidation < ValidationExample + def params + [:algorithm_uri, :training_dataset_uri, :test_dataset_uri, :prediction_feature] + end + + def opt_params + [ :algorithm_params, :test_target_dataset_uri ] + end + end + + class SplitTestValidation < ValidationExample + def params + [:algorithm_uri, :dataset_uri, :prediction_feature] + end + + def opt_params + [ :algorithm_params, :split_ratio, :random_seed ] + end + + def validation_type + "training_test_split" + end + end + + class CrossValidation < ValidationExample + def params + [:algorithm_uri, :dataset_uri, :prediction_feature] + end + + def opt_params + [ :algorithm_params, :num_folds, :stratified, :random_seed ] + end + + def report_type + "crossvalidation" + end + + def validation_type + "crossvalidation" + end + end +end
\ No newline at end of file |