From eeb0b6df2dbdae29ccf3f4ccd923002e0ed65506 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 13 Jan 2011 15:18:45 +0100 Subject: adjust to new wrapper, replace activerecord with datamapper (remove activerecord code), is_classification to feature_type --- db/migrate/000_drop_validations.rb | 0 db/migrate/001_init_validation.rb | 0 db/migrate/002_init_reports.rb | 4 +- .../.CatalogManager.properties.example | 0 docbook-xsl-1.76.1/.urilist | 0 example.rb | 56 +++-- lib/active_record_setup.rb | 19 +- lib/ot_predictions.rb | 136 +++++++---- lib/predictions.rb | 51 ++-- lib/test_util.rb | 4 +- lib/validation_db.rb | 65 ++++- nightly/nightly.rb | 8 +- reach_reports/reach_application.rb | 4 +- reach_reports/reach_persistance.rb | 44 +--- reach_reports/reach_service.rb | 59 ++--- reach_reports/reach_test.rb | 12 +- report/environment.rb | 7 +- report/report_application.rb | 4 +- report/report_content.rb | 6 +- report/report_factory.rb | 23 +- report/report_persistance.rb | 109 +++++++-- report/report_test.rb | 13 +- report/validation_access.rb | 58 +++-- report/validation_data.rb | 72 +++--- report/xml_report.rb | 4 +- report/xml_report_util.rb | 9 +- test/test_application.rb | 3 +- test/test_examples.rb | 114 ++++++--- test/test_examples_util.rb | 96 ++++++-- validation/validation_application.rb | 161 ++++++++----- validation/validation_format.rb | 6 +- validation/validation_service.rb | 265 ++++++++++++++------- validation/validation_test.rb | 49 +++- 33 files changed, 959 insertions(+), 502 deletions(-) mode change 100644 => 100755 db/migrate/000_drop_validations.rb mode change 100644 => 100755 db/migrate/001_init_validation.rb mode change 100644 => 100755 db/migrate/002_init_reports.rb mode change 100644 => 100755 docbook-xsl-1.76.1/.CatalogManager.properties.example mode change 100644 => 100755 docbook-xsl-1.76.1/.urilist mode change 100644 => 100755 example.rb mode change 100644 => 100755 lib/active_record_setup.rb mode change 100644 => 100755 lib/ot_predictions.rb mode change 100644 => 100755 lib/predictions.rb mode change 100644 => 100755 lib/test_util.rb mode change 100644 => 100755 lib/validation_db.rb mode change 100644 => 100755 nightly/nightly.rb mode change 100644 => 100755 reach_reports/reach_application.rb mode change 100644 => 100755 reach_reports/reach_persistance.rb mode change 100644 => 100755 reach_reports/reach_service.rb mode change 100644 => 100755 reach_reports/reach_test.rb mode change 100644 => 100755 report/environment.rb mode change 100644 => 100755 report/report_application.rb mode change 100644 => 100755 report/report_content.rb mode change 100644 => 100755 report/report_factory.rb mode change 100644 => 100755 report/report_persistance.rb mode change 100644 => 100755 report/report_test.rb mode change 100644 => 100755 report/validation_access.rb mode change 100644 => 100755 report/validation_data.rb mode change 100644 => 100755 report/xml_report.rb mode change 100644 => 100755 report/xml_report_util.rb mode change 100644 => 100755 test/test_application.rb mode change 100644 => 100755 test/test_examples.rb mode change 100644 => 100755 test/test_examples_util.rb mode change 100644 => 100755 validation/validation_application.rb mode change 100644 => 100755 validation/validation_format.rb mode change 100644 => 100755 validation/validation_service.rb mode change 100644 => 100755 validation/validation_test.rb diff --git a/db/migrate/000_drop_validations.rb b/db/migrate/000_drop_validations.rb old mode 100644 new mode 100755 diff --git a/db/migrate/001_init_validation.rb b/db/migrate/001_init_validation.rb old mode 100644 new mode 100755 diff --git a/db/migrate/002_init_reports.rb b/db/migrate/002_init_reports.rb old mode 100644 new mode 100755 index 8029223..dcf44a1 --- a/db/migrate/002_init_reports.rb +++ b/db/migrate/002_init_reports.rb @@ -22,9 +22,9 @@ class InitReports < ActiveRecord::Migration def self.down drop_table :report_datum if table_exists? :report_datum - if @@config[:reports] and @@config[:reports][:report_dir] + if CONFIG[:reports] and CONFIG[:reports][:report_dir] ["validation", "crossvalidation", "algorithm_comparison"].each do |t| - dir = File.join(@@config[:reports][:report_dir],t) + dir = File.join(CONFIG[:reports][:report_dir],t) if File.exist?(dir) puts "deleting dir "+dir.to_s FileUtils.rm_rf(dir) diff --git a/docbook-xsl-1.76.1/.CatalogManager.properties.example b/docbook-xsl-1.76.1/.CatalogManager.properties.example old mode 100644 new mode 100755 diff --git a/docbook-xsl-1.76.1/.urilist b/docbook-xsl-1.76.1/.urilist old mode 100644 new mode 100755 diff --git a/example.rb b/example.rb old mode 100644 new mode 100755 index 3562dd0..1b4ee98 --- a/example.rb +++ b/example.rb @@ -5,15 +5,16 @@ class Example @@file=File.new("data/hamster_carcinogenicity.yaml","r") @@file_type="text/x-yaml" - @@model=File.join @@config[:services]["opentox-model"],"1" - @@feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)") - @@predicted_feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)_lazar_classification") - @@alg = File.join @@config[:services]["opentox-algorithm"],"lazar" - @@alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") - @@data=File.join @@config[:services]["opentox-dataset"],"1" - @@train_data=File.join @@config[:services]["opentox-dataset"],"2" - @@test_data=File.join @@config[:services]["opentox-dataset"],"3" - @@prediction_data=File.join @@config[:services]["opentox-dataset"],"5" + @@model=File.join CONFIG[:services]["opentox-model"],"1" + #@@feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)") + @@feature= File.join CONFIG[:services]["opentox-dataset"],"1/feature/hamster_carcinogenicity" + #@@predicted_feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)_lazar_classification") + @@alg = File.join CONFIG[:services]["opentox-algorithm"],"lazar" + @@alg_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + @@data=File.join CONFIG[:services]["opentox-dataset"],"1" + @@train_data=File.join CONFIG[:services]["opentox-dataset"],"2" + @@test_data=File.join CONFIG[:services]["opentox-dataset"],"3" + @@prediction_data=File.join CONFIG[:services]["opentox-dataset"],"5" @@css_file="http://apps.ideaconsult.net:8080/ToxPredict/style/global.css" @@summary="" @@ -28,7 +29,7 @@ class Example end file.close - sub = { "validation_service" => @@config[:services]["opentox-validation"].chomp("/"), + sub = { "validation_service" => CONFIG[:services]["opentox-validation"].chomp("/"), "validation_id" => "1", "model_uri" => @@model, "dataset_uri" => @@data, @@ -42,7 +43,7 @@ class Example "crossvalidation_report_id" => "2", "css_file" => @@css_file, "prediction_dataset_uri" => @@prediction_data, - "predicted_feature" => @@predicted_feature, + #"predicted_feature" => @@predicted_feature, "qmrf_id" => "1"} sub.each do |k,v| @@ -53,31 +54,39 @@ class Example # creates the resources that are requested by the examples def self.prepare_example_resources - OpenTox::Task.as_task("prepare examples", "n/a") do |task| + task = OpenTox::Task.create("prepare examples", "n/a") do |task| @@summary = "" #delete validations log "delete validations" - ActiveRecord::Base.logger = Logger.new("/dev/null") - ActiveRecord::Migrator.migrate('db/migrate', 0 ) - ActiveRecord::Migrator.migrate('db/migrate', 1 ) - ActiveRecord::Migrator.migrate('db/migrate', 2 ) + Lib::Validation.auto_migrate! + Lib::Crossvalidation.auto_migrate! + #ActiveRecord::Base.logger = Logger.new("/dev/null") + #ActiveRecord::Migrator.migrate('db/migrate', 0 ) + #ActiveRecord::Migrator.migrate('db/migrate', 1 ) + #ActiveRecord::Migrator.migrate('db/migrate', 2 ) #delete all qmrf reports ReachReports::QmrfReport.auto_migrate! - #delete_all(@@config[:services]["opentox-dataset"]) - log OpenTox::RestClientWrapper.delete @@config[:services]["opentox-dataset"] + #delete_all(CONFIG[:services]["opentox-dataset"]) + log OpenTox::RestClientWrapper.delete CONFIG[:services]["opentox-dataset"] task.progress(10) log "upload dataset" halt 400,"File not found: "+@@file.path.to_s unless File.exist?(@@file.path) + #data = File.read(@@file.path) + #data_uri = OpenTox::RestClientWrapper.post(CONFIG[:services]["opentox-dataset"],{:content_type => @@file_type},data).chomp("\n") data = File.read(@@file.path) - data_uri = OpenTox::RestClientWrapper.post(@@config[:services]["opentox-dataset"],{:content_type => @@file_type},data).chomp("\n") + dataset = OpenTox::Dataset.create + dataset.load_yaml(data) + dataset.save + data_uri = dataset.uri + log "-> "+data_uri task.progress(20) log "train-test-validation" - #delete_all(@@config[:services]["opentox-model"]) - OpenTox::RestClientWrapper.delete @@config[:services]["opentox-model"] + #delete_all(CONFIG[:services]["opentox-model"]) + OpenTox::RestClientWrapper.delete CONFIG[:services]["opentox-model"] split_params = Validation::Util.train_test_dataset_split(data_uri, URI.decode(@@feature), 0.9, 1) v = Validation::Validation.new :training_dataset_uri => split_params[:training_dataset_uri], @@ -93,7 +102,7 @@ class Example cv.perform_cv( URI.decode(@@feature), @@alg_params, OpenTox::SubTask.new(task, 40, 70) ) log "create validation report" - rep = Reports::ReportService.instance(File.join(@@config[:services]["opentox-validation"],"report")) + rep = Reports::ReportService.instance(File.join(CONFIG[:services]["opentox-validation"],"report")) rep.delete_all_reports("validation") rep.create_report("validation",v.validation_uri) task.progress(80) @@ -111,6 +120,7 @@ class Example log "done" @@summary end + task.uri end # performs all curl calls listed in examples after ">>>", next line is added if line ends with "\" @@ -163,7 +173,7 @@ class Example result.chomp! result.gsub!(/\n/, " \\n ") if ($?==0) - if OpenTox::Utils.task_uri?(result) + if result.task_uri? log "wait for task: "+result result = Lib::TestUtil.wait_for_task(result) end diff --git a/lib/active_record_setup.rb b/lib/active_record_setup.rb old mode 100644 new mode 100755 index 3682c7a..404bfec --- a/lib/active_record_setup.rb +++ b/lib/active_record_setup.rb @@ -1,17 +1,17 @@ -gem "activerecord", "= 2.3.8" -gem "ar-extensions", "= 0.9.2" -['rubygems', 'logger', 'active_record', 'ar-extensions', 'opentox-ruby-api-wrapper' ].each do |g| +#gem "activerecord", "= 2.3.8" +#gem "ar-extensions", "= 0.9.2" +['rubygems', 'logger', 'active_record', 'opentox-ruby' ].each do |g| #'ar-extensions', require g end unless ActiveRecord::Base.connected? ActiveRecord::Base.establish_connection( - :adapter => @@config[:database][:adapter], - :host => @@config[:database][:host], - :database => @@config[:database][:database], - :username => @@config[:database][:username], - :password => @@config[:database][:password] + :adapter => CONFIG[:database][:adapter], + :host => CONFIG[:database][:host], + :database => CONFIG[:database][:database], + :username => CONFIG[:database][:username], + :password => CONFIG[:database][:password] ) ActiveRecord::Base.logger = Logger.new("/dev/null") end @@ -19,6 +19,9 @@ end class ActiveRecord::Base def self.find_like(filter_params) + + raise "find like removed" + #puts "params before "+filter_params.inspect filter_params.keys.each do |k| key = k.to_s diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb old mode 100644 new mode 100755 index 0175a0c..52683a6 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -15,7 +15,7 @@ module Lib return @compounds[instance_index] end - def initialize(is_classification, test_dataset_uri, test_target_dataset_uri, + def initialize(feature_type, test_dataset_uri, test_target_dataset_uri, prediction_feature, prediction_dataset_uri, predicted_variable, task=nil) LOGGER.debug("loading prediciton via test-dataset:'"+test_dataset_uri.to_s+ @@ -53,38 +53,47 @@ module Lib raise "prediction_feature not found in test_target_dataset\n"+ "prediction_feature: '"+prediction_feature.to_s+"'\n"+ "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ - "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.index(prediction_feature)==nil + "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil end + test_dataset.load_all @compounds = test_dataset.compounds LOGGER.debug "test dataset size: "+@compounds.size.to_s raise "test dataset is empty" unless @compounds.size>0 - class_values = is_classification ? OpenTox::Feature.domain(prediction_feature) : nil + class_values = feature_type=="classification" ? OpenTox::Feature.new(prediction_feature).domain : nil actual_values = [] @compounds.each do |c| - value = test_target_dataset.get_value(c, prediction_feature) - - if is_classification - value = value.to_s unless value==nil - raise "illegal class_value of actual value "+value.to_s+" class: "+ - value.class.to_s unless value==nil or class_values.index(value)!=nil - actual_values.push class_values.index(value) - else - begin - value = value.to_f unless value==nil or value.is_a?(Numeric) - rescue - LOGGER.warn "no numeric value for regression: '"+value.to_s+"'" - value = nil - end - actual_values.push value + case feature_type + when "classification" + actual_values << classification_value(test_target_dataset, c, prediction_feature, class_values) + when "regression" + actual_values << regression_value(test_target_dataset, c, prediction_feature) end end task.progress(40) if task # loaded actual values prediction_dataset = OpenTox::Dataset.find prediction_dataset_uri raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset - raise "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+prediction_dataset.features.inspect if prediction_dataset.features.index(predicted_variable)==nil + + # TODO: remove LAZAR_PREDICTION_DATASET_HACK + no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil + if no_prediction_feature + one_entry_per_compound = prediction_dataset.data_entries.keys.size == @compounds.size + @compounds.each do |c| + if prediction_dataset.data_entries[c].size != 1 + one_entry_per_compound = false + break + end + end + msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+ + prediction_dataset.features.keys.inspect + if one_entry_per_compound + LOGGER.warn msg + else + raise msg + end + end raise "more predicted than test compounds test:"+@compounds.size.to_s+" < prediction:"+ prediction_dataset.compounds.size.to_s if @compounds.size < prediction_dataset.compounds.size @@ -102,38 +111,79 @@ module Lib predicted_values << nil confidence_values << nil else - if is_classification - value = prediction_dataset.get_predicted_class(c, predicted_variable) - value = value.to_s unless value==nil - raise "illegal class_value of predicted value "+value.to_s+" class: "+value.class.to_s unless value==nil or class_values.index(value)!=nil - predicted_values << class_values.index(value) - else - value = prediction_dataset.get_predicted_regression(c, predicted_variable) - begin - value = value.to_f unless value==nil or value.is_a?(Numeric) - rescue - LOGGER.warn "no numeric value for regression: '"+value.to_s+"'" - value = nil - end - predicted_values << value + case feature_type + when "classification" + # TODO: remove LAZAR_PREDICTION_DATASET_HACK + predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, class_values) + when "regression" + predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable) + end + # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable) + conf = 1 + begin + feature = prediction_dataset.data_entries[c].keys[0] + feature_data = prediction_dataset.features[feature] + conf = feature_data[OT.confidence] if feature_data[OT.confidence]!=nil + rescue + LOGGER.warn "could not get confidence" end - confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable) + confidence_values << conf end end task.progress(80) if task # loaded predicted values and confidence - super(predicted_values, actual_values, confidence_values, is_classification, class_values) + super(predicted_values, actual_values, confidence_values, feature_type, class_values) raise "illegal num compounds "+num_info if @compounds.size != @predicted_values.size task.progress(100) if task # done with the mathmatics end + private + def regression_value(dataset, compound, feature) + v = value(dataset, compound, feature) + begin + v = v.to_f unless v==nil or v.is_a?(Numeric) + v + rescue + LOGGER.warn "no numeric value for regression: '"+v.to_s+"'" + nil + end + end + + def classification_value(dataset, compound, feature, class_values) + v = value(dataset, compound, feature) + i = class_values.index(v) + raise "illegal class_value of predicted value "+v.to_s+" class: "+v.class.to_s unless v==nil or i!=nil + i + end + + def value(dataset, compound, feature) + + if feature==nil + v = dataset.data_entries[compound].values[0] + else + v = dataset.data_entries[compound][feature] + end + raise "no array" unless v.is_a?(Array) + if v.size>1 + raise "multiple values" + elsif v.size==1 + v = v[0] + else + v = nil + end + raise "array" if v.is_a?(Array) + v = nil if v.to_s.size==0 + v + end + public def compute_stats res = {} - if @is_classification + case @feature_type + when "classification" (Lib::VAL_CLASS_PROPS).each{ |s| res[s] = send(s)} - else + when "regression" (Lib::VAL_REGR_PROPS).each{ |s| res[s] = send(s) } end return res @@ -152,16 +202,18 @@ module Lib #PENDING! begin - a.push( "http://ambit.uni-plovdiv.bg:8080/ambit2/depict/cdk?search="+ - URI.encode(OpenTox::Compound.new(:uri=>p.identifier(i)).smiles) ) if add_pic + #a.push( "http://ambit.uni-plovdiv.bg:8080/ambit2/depict/cdk?search="+ + # URI.encode(OpenTox::Compound.new(:uri=>p.identifier(i)).smiles) ) if add_pic + a << p.identifier(i)+"/image" rescue => ex + raise ex #a.push("Could not add pic: "+ex.message) - a.push(p.identifier(i)) + #a.push(p.identifier(i)) end a << (format ? p.actual_value(i).to_nice_s : p.actual_value(i)) a << (format ? p.predicted_value(i).to_nice_s : p.predicted_value(i)) - if p.classification? + if p.feature_type=="classification" if (p.predicted_value(i)!=nil and p.actual_value(i)!=nil) a << (p.classification_miss?(i) ? 1 : 0) else @@ -180,7 +232,7 @@ module Lib header << "compound" if add_pic header << "actual value" header << "predicted value" - header << "missclassified" if predictions[0].classification? + header << "missclassified" if predictions[0].feature_type=="classification" header << "confidence value" if predictions[0].confidence_values_available? header << "compound-uri" res.insert(0, header) diff --git a/lib/predictions.rb b/lib/predictions.rb old mode 100644 new mode 100755 index 6e50e94..e73dda6 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -22,13 +22,13 @@ module Lib def initialize( predicted_values, actual_values, confidence_values, - is_classification, + feature_type, class_domain=nil ) @predicted_values = predicted_values @actual_values = actual_values @confidence_values = confidence_values - @is_classification = is_classification + @feature_type = feature_type @class_domain = class_domain @num_classes = 1 @@ -36,6 +36,8 @@ module Lib #puts "actual: "+actual_values.inspect #puts "confidence: "+confidence_values.inspect + raise "unknown feature_type: "+@feature_type.to_s unless + @feature_type=="classification" || @feature_type=="regression" raise "no predictions" if @predicted_values.size == 0 num_info = "predicted:"+@predicted_values.size.to_s+ " confidence:"+@confidence_values.size.to_s+" actual:"+@actual_values.size.to_s @@ -54,15 +56,16 @@ module Lib # @confidence_values=nil #end - if @is_classification + case @feature_type + when "classification" raise "class_domain missing while performing classification" unless @class_domain @num_classes = @class_domain.size raise "num classes < 2" if @num_classes<2 { "predicted"=>@predicted_values, "actual"=>@actual_values }.each do |s,values| values.each{ |v| raise "illegal "+s+" classification-value ("+v.to_s+"),"+ - "has to be either nil or index of predicted-values" if v!=nil and (v<0 or v>@num_classes)} + "has to be either nil or index of predicted-values" if v!=nil and (!v.is_a?(Numeric) or v<0 or v>@num_classes)} end - else + when "regresssion" raise "class_domain != nil while performing regression" if @class_domain { "predicted"=>@predicted_values, "actual"=>@actual_values }.each do |s,values| values.each{ |v| raise "illegal "+s+" regression-value ("+v.to_s+"),"+ @@ -83,7 +86,8 @@ module Lib @num_predicted = 0 @num_unpredicted = 0 - if @is_classification + case @feature_type + when "classification" @confusion_matrix = [] @class_domain.each do |v| @confusion_matrix.push( Array.new( @num_classes, 0 ) ) @@ -91,7 +95,7 @@ module Lib @num_correct = 0 @num_incorrect = 0 - else + when "regression" @sum_error = 0 @sum_abs_error = 0 @sum_squared_error = 0 @@ -122,14 +126,15 @@ module Lib else @num_predicted += 1 - if @is_classification + case @feature_type + when "classification" @confusion_matrix[actual_value][predicted_value] += 1 if (predicted_value == actual_value) @num_correct += 1 else @num_incorrect += 1 end - else + when "regression" delta = predicted_value - actual_value @sum_error += delta @sum_abs_error += delta.abs @@ -156,13 +161,13 @@ module Lib end def percent_correct - raise "no classification" unless @is_classification + raise "no classification" unless @feature_type=="classification" return 0 if @num_with_actual_value==0 return 100 * @num_correct / @num_with_actual_value.to_f end def percent_incorrect - raise "no classification" unless @is_classification + raise "no classification" unless @feature_type=="classification" return 0 if @num_with_actual_value==0 return 100 * @num_incorrect / @num_with_actual_value.to_f end @@ -190,17 +195,17 @@ module Lib end def num_correct - raise "no classification" unless @is_classification + raise "no classification" unless @feature_type=="classification" return @num_correct end def num_incorrect - raise "no classification" unless @is_classification + raise "no classification" unless @feature_type=="classification" return @num_incorrect end def num_unclassified - raise "no classification" unless @is_classification + raise "no classification" unless @feature_type=="classification" return @num_unpredicted end @@ -209,7 +214,7 @@ module Lib # and values: def confusion_matrix - raise "no classification" unless @is_classification + raise "no classification" unless @feature_type=="classification" res = {} (0..@num_classes-1).each do |actual| (0..@num_classes-1).each do |predicted| @@ -505,9 +510,10 @@ module Lib end def predicted_value(instance_index) - if @is_classification + case @feature_type + when "classification" @predicted_values[instance_index]==nil ? nil : @class_domain[@predicted_values[instance_index]] - else + when "regression" @predicted_values[instance_index] end end @@ -517,9 +523,10 @@ module Lib end def actual_value(instance_index) - if @is_classification + case @feature_type + when "classification" @actual_values[instance_index]==nil ? nil : @class_domain[@actual_values[instance_index]] - else + when "regression" @actual_values[instance_index] end end @@ -529,13 +536,13 @@ module Lib end def classification_miss?(instance_index) - raise "no classification" unless @is_classification + raise "no classification" unless @feature_type=="classification" return false if predicted_value(instance_index)==nil or actual_value(instance_index)==nil return predicted_value(instance_index) != actual_value(instance_index) end - def classification? - @is_classification + def feature_type + @feature_type end def confidence_values_available? diff --git a/lib/test_util.rb b/lib/test_util.rb old mode 100644 new mode 100755 index ecab76c..0deee04 --- a/lib/test_util.rb +++ b/lib/test_util.rb @@ -10,11 +10,11 @@ module Lib end def self.wait_for_task(uri) - if OpenTox::Utils.task_uri?(uri) + if uri.task_uri? task = OpenTox::Task.find(uri) task.wait_for_completion raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error? - uri = task.resultURI + uri = task.result_uri end return uri end diff --git a/lib/validation_db.rb b/lib/validation_db.rb old mode 100644 new mode 100755 index cb3ece7..83b7e2f --- a/lib/validation_db.rb +++ b/lib/validation_db.rb @@ -49,11 +49,40 @@ module Lib VAL_MERGE_AVG = VAL_PROPS_AVG + VAL_CLASS_PROPS_SINGLE_AVG + VAL_CLASS_PROPS_PER_CLASS_AVG + VAL_REGR_PROPS - class Validation < ActiveRecord::Base - serialize :classification_statistics - serialize :regression_statistics +# class Validation < ActiveRecord::Base +# serialize :classification_statistics +# serialize :regression_statistics +# +# alias_attribute :date, :created_at - alias_attribute :date, :created_at + class Validation + include DataMapper::Resource + + property :id, Serial + property :validation_type, String, :length => 255 + property :model_uri, String, :length => 255 + property :algorithm_uri, String, :length => 255 + property :training_dataset_uri, String, :length => 255 + property :test_target_dataset_uri, String, :length => 255 + property :test_dataset_uri, String, :length => 255 + property :prediction_dataset_uri, String, :length => 255 + property :prediction_feature, String, :length => 255 + property :created_at, DateTime + property :num_instances, Integer + property :num_without_class, Integer + property :num_unpredicted, Integer + property :crossvalidation_id, Integer + property :crossvalidation_fold, Integer + property :real_runtime, Float + property :percent_without_class, Float + property :percent_unpredicted, Float + property :classification_statistics, Object + property :regression_statistics, Object + property :finished, Boolean, :default => false + + def date + created_at + end def validation_uri $sinatra.url_for("/"+self.id.to_s, :full) @@ -77,8 +106,23 @@ module Lib end - class Crossvalidation < ActiveRecord::Base - alias_attribute :date, :created_at +# class Crossvalidation < ActiveRecord::Base +# alias_attribute :date, :created_at + class Crossvalidation + include DataMapper::Resource + + property :id, Serial + property :algorithm_uri, String, :length => 255 + property :dataset_uri, String, :length => 255 + property :created_at, DateTime + property :num_folds, Integer, :default => 10 + property :random_seed, Integer, :default => 1 + property :finished, Boolean, :default => false + property :stratified, Boolean, :default => false + + def date + created_at + end def crossvalidation_uri $sinatra.url_for("/crossvalidation/"+self.id.to_s, :full) if self.id @@ -88,7 +132,8 @@ module Lib # in terms of dataset_uri,num_folds,stratified,random_seed # further conditions can be specified in __conditions__ def self.find_all_uniq(conditions={}) - cvs = Lib::Crossvalidation.find(:all, :conditions => conditions) + #cvs = Lib::Crossvalidation.find(:all, :conditions => conditions) + cvs = Lib::Crossvalidation.all(:conditions => conditions) uniq = [] cvs.each do |cv| match = false @@ -105,3 +150,9 @@ module Lib end end end + + +Lib::Validation.auto_upgrade! +Lib::Validation.raise_on_save_failure = true +Lib::Crossvalidation.auto_upgrade! +Lib::Crossvalidation.raise_on_save_failure = true \ No newline at end of file diff --git a/nightly/nightly.rb b/nightly/nightly.rb old mode 100644 new mode 100755 index eb802a3..550d14c --- a/nightly/nightly.rb +++ b/nightly/nightly.rb @@ -21,7 +21,7 @@ class Nightly validationExamples = ValidationExamples.select(select) return "please \"select\" validation examples:\n"+ValidationExamples.list if validationExamples.size==0 - task_uri = OpenTox::Task.as_task("Build nightly","nightly-validation-test-service",{:select => select, :dry_run => dry_run}) do |task| + task = OpenTox::Task.create("Build nightly","nightly-validation-test-service") do |task| #,{:select => select, :dry_run => dry_run}) LOGGER.info("Building nightly report") benchmarks = validationExamples.collect{ |e| ValidationBenchmark.new(e) } @@ -98,12 +98,12 @@ class Nightly end #benchmarks.collect{|b| b.uris}.join(",") - File.join(@@config[:services]["opentox-validation"],"nightly") + File.join(CONFIG[:services]["opentox-validation"],"nightly") end if defined?(halt) - halt 202,task_uri+"\n" + halt 202,task.uri+"\n" else - return task_uri+"\n" + return task.uri+"\n" end end diff --git a/reach_reports/reach_application.rb b/reach_reports/reach_application.rb old mode 100644 new mode 100755 index d82bc11..4187c1b --- a/reach_reports/reach_application.rb +++ b/reach_reports/reach_application.rb @@ -1,5 +1,5 @@ -[ 'rubygems', 'sinatra', 'sinatra/url_for', 'opentox-ruby-api-wrapper' ].each do |lib| +[ 'rubygems', 'sinatra', 'sinatra/url_for', 'opentox-ruby' ].each do |lib| require lib end @@ -67,7 +67,7 @@ post '/reach_report/:type' do #puts "creating "+type+" report "+params.inspect result_uri = ReachReports.create_report(type,params,request.env["rack.input"]) - if OpenTox::Utils.task_uri?(result_uri) + if result_uri and result_uri.task_uri? halt 202,result_uri+"\n" else result_uri+"\n" diff --git a/reach_reports/reach_persistance.rb b/reach_reports/reach_persistance.rb old mode 100644 new mode 100755 index 2cbcd71..a36e05f --- a/reach_reports/reach_persistance.rb +++ b/reach_reports/reach_persistance.rb @@ -1188,41 +1188,11 @@ module ReachReports # end # end - QsarSoftware.auto_upgrade! - QsarIdentifier.auto_upgrade! - - QmrfAuthor.auto_upgrade! - ModelAuthor.auto_upgrade! - Reference.auto_upgrade! - QsarGeneralInformation.auto_upgrade! - - ModelEndpoint.auto_upgrade! - QsarEndpoint.auto_upgrade! - - AlgorithmExplicit.auto_upgrade! - AlgorithmsDescriptor.auto_upgrade! - DescriptorsGenerationSoftware.auto_upgrade! - QsarAlgorithm.auto_upgrade! - - AppDomainSoftware.auto_upgrade! - QsarApplicabilityDomain.auto_upgrade! - - TrainingSetData.auto_upgrade! - QsarRobustness.auto_upgrade! - - ValidationSetData.auto_upgrade! - QsarPredictivity.auto_upgrade! - - QsarInterpretation.auto_upgrade! - - Bibliography.auto_upgrade! - AttachmentTrainingData.auto_upgrade! - AttachmentValidationData.auto_upgrade! - AttachmentDocument.auto_upgrade! - QsarMiscellaneous.auto_upgrade! - - QmrfSummary.auto_upgrade! - - QmrfReport.auto_upgrade! - + [ QsarSoftware, QsarIdentifier, QmrfAuthor, ModelAuthor, Reference, QsarGeneralInformation, ModelEndpoint, QsarEndpoint, AlgorithmExplicit, + AlgorithmsDescriptor, DescriptorsGenerationSoftware, QsarAlgorithm, AppDomainSoftware, QsarApplicabilityDomain, TrainingSetData, + QsarRobustness, ValidationSetData, QsarPredictivity, QsarInterpretation, Bibliography, AttachmentTrainingData, AttachmentValidationData, + AttachmentDocument, QsarMiscellaneous, QmrfSummary, QmrfReport ].each do |model| + model.auto_upgrade! + model.raise_on_save_failure = true + end end \ No newline at end of file diff --git a/reach_reports/reach_service.rb b/reach_reports/reach_service.rb old mode 100644 new mode 100755 index f297f26..808d08a --- a/reach_reports/reach_service.rb +++ b/reach_reports/reach_service.rb @@ -26,13 +26,14 @@ module ReachReports case type when /(?i)QMRF/ if params[:model_uri] - result_uri = OpenTox::Task.as_task( "Create "+type+" report", - $sinatra.url_for("/reach_report/"+type, :full), params ) do |task| + task = OpenTox::Task.create( "Create "+type+" report", + $sinatra.url_for("/reach_report/"+type, :full) ) do |task| #, params report = ReachReports::QmrfReport.new :model_uri => params[:model_uri] build_qmrf_report(report, task) report.report_uri end + result_uri = task.uri elsif xml_data and (input = xml_data.read).to_s.size>0 report = ReachReports::QmrfReport.new ReachReports::QmrfReport.from_xml(report,input) @@ -61,17 +62,18 @@ module ReachReports def self.build_qmrf_report(r, task=nil) #puts r.model_uri - model = OpenTox::Model::PredictionModel.find(r.model_uri) - classification = model.classification? + model = OpenTox::Model::Generic.find(r.model_uri) + raise "model not found "+r.model_uri.to_s unless model + feature_type = model.feature_type # chapter 1 r.qsar_identifier = QsarIdentifier.new - r.qsar_identifier.qsar_title = model.title + r.qsar_identifier.qsar_title = model.metadata[DC.title] # TODO QSAR_models -> sparql same endpoint r.qsar_identifier.qsar_software << QsarSoftware.new( :url => model.uri, - :name => model.title, :contact => model.creator ) - algorithm = OpenTox::Algorithm::Generic.find(model.algorithm) if model.algorithm - r.qsar_identifier.qsar_software << QsarSoftware.new( :url => algorithm.uri, :name => algorithm.title ) + :name => model.metadata[DC.title], :contact => model.metadata[DC.creator] ) + algorithm = OpenTox::Algorithm::Generic.find(model.metadata[OT.algorithm]) if model.metadata[OT.algorithm] + r.qsar_identifier.qsar_software << QsarSoftware.new( :url => algorithm.uri, :name => algorithm.metadata[DC.title] ) task.progress(10) if task #chpater 2 @@ -79,7 +81,7 @@ module ReachReports r.qsar_general_information.qmrf_date = DateTime.now.to_s # EMPTY: qmrf_authors, qmrf_date_revision, qmrf_revision # TODO: model_authors ? - r.qsar_general_information.model_date = model.date.to_s + r.qsar_general_information.model_date = model.metadata[DC.date].to_s # TODO: references? # EMPTY: info_availablity # TODO: related_models = find qmrf reports for QSAR_models @@ -88,11 +90,11 @@ module ReachReports # chapter 3 # TODO "model_species" ? r.qsar_endpoint = QsarEndpoint.new - model.predictedVariables.each do |p| + model.metadata[OT.predictedVariables].each do |p| r.qsar_endpoint.model_endpoint << ModelEndpoint.new( :name => p ) - end + end if model.metadata[OT.predictedVariables] # TODO "endpoint_comments" => "3.3", "endpoint_units" => "3.4", - r.qsar_endpoint.endpoint_variable = model.dependentVariables if model.dependentVariables + r.qsar_endpoint.endpoint_variable = model.metadata[OT.dependentVariables] if model.metadata[OT.dependentVariables] # TODO "endpoint_protocol" => "3.6", "endpoint_data_quality" => "3.7", task.progress(30) if task @@ -108,9 +110,9 @@ module ReachReports #training_dataset = model.trainingDataset ? OpenTox::Dataset.find(model.trainingDataset+"/metadata") : nil begin - training_dataset = model.trainingDataset ? OpenTox::Dataset.find(model.trainingDataset) : nil + training_dataset = model.metadata[OT.trainingDataset] ? OpenTox::Dataset.find(model.metadata[OT.trainingDataset]) : nil rescue - LOGGER.warn "build qmrf: training_dataset not found "+model.trainingDataset.to_s + LOGGER.warn "build qmrf: training_dataset not found "+model.metadata[OT.trainingDataset].to_s end task.progress(50) if task @@ -129,27 +131,27 @@ module ReachReports val_datasets = [] - if model.algorithm - cvs = Lib::Crossvalidation.find_all_uniq({:algorithm_uri => model.algorithm}) + if algorithm + cvs = Lib::Crossvalidation.find_all_uniq({:algorithm_uri => algorithm.uri}) # PENDING: cv classification/regression hack cvs = cvs.delete_if do |cv| val = Validation::Validation.first( :all, :conditions => { :crossvalidation_id => cv.id } ) - (val.classification_statistics!=nil) != classification + (val.classification_statistics!=nil) != (feature_type=="classification") end - lmo = [ "found "+cvs.size.to_s+" crossvalidation/s for algorithm '"+model.algorithm+"'" ] + lmo = [ "found "+cvs.size.to_s+" crossvalidation/s for algorithm '"+algorithm.uri.to_s+"'" ] if cvs.size>0 cvs_same_data = [] cvs_other_data = [] cvs.each do |cv| - if cv.dataset_uri == model.trainingDataset + if cv.dataset_uri == model.metadata[OT.trainingDataset] cvs_same_data << cv else cvs_other_data << cv end end lmo << cvs_same_data.size.to_s+" crossvalidations/s where performed on the training dataset of the model ("+ - model.trainingDataset.to_s+")" + model.metadata[OT.trainingDataset].to_s+")" lmo << cvs_other_data.size.to_s+" crossvalidations/s where performed on the other datasets" lmo << "" @@ -162,15 +164,16 @@ module ReachReports val_datasets << cv.dataset_uri lmo << "settings: num-folds="+cv.num_folds.to_s+", random-seed="+cv.random_seed.to_s+", stratified:"+cv.stratified.to_s val = YAML.load( OpenTox::RestClientWrapper.get File.join(cv.crossvalidation_uri,"statistics") ) - if classification + case feature_type + when "classification" lmo << "percent_correct: "+val[:classification_statistics][:percent_correct].to_s lmo << "weighted AUC: "+val[:classification_statistics][:weighted_area_under_roc].to_s - else + when "regression" lmo << "root_mean_squared_error: "+val[:regression_statistics][:root_mean_squared_error].to_s lmo << "r_square "+val[:regression_statistics][:r_square].to_s end reports = OpenTox::RestClientWrapper.get File.join(CONFIG[:services]["opentox-validation"],"report/crossvalidation?crossvalidation_uris="+cv.crossvalidation_uri) - if reports and reports.size>0 + if reports and reports.chomp.size>0 lmo << "for more info see report: "+reports.split("\n")[0] else lmo << "for more info see report: not yet created for '"+cv.crossvalidation_uri+"'" @@ -187,7 +190,8 @@ module ReachReports # "lmo" => "6.9", "yscrambling" => "6.10", "bootstrap" => "6.11", "other_statistics" => "6.12", LOGGER.debug "looking for validations with "+{:model_uri => model.uri}.inspect - vals = Lib::Validation.find(:all, :conditions => {:model_uri => model.uri}) + #vals = Lib::Validation.find(:all, :conditions => {:model_uri => model.uri}) + vals = Lib::Validation.all({:model_uri => model.uri}) uniq_vals = [] vals.each do |val| match = false @@ -212,10 +216,11 @@ module ReachReports v << "validation: "+validation.validation_uri v << "dataset (see 9.3 Validation data): "+validation.test_dataset_uri val_datasets << validation.test_dataset_uri - if classification + case feature_type + when "classification" v << "percent_correct: "+validation.classification_statistics[:percent_correct].to_s v << "weighted AUC: "+validation.classification_statistics[:weighted_area_under_roc].to_s - else + when "regression" v << "root_mean_squared_error: "+validation.regression_statistics[:root_mean_squared_error].to_s v << "r_square "+validation.regression_statistics[:r_square].to_s end @@ -252,7 +257,7 @@ module ReachReports r.qsar_miscellaneous.attachment_training_data << AttachmentTrainingData.new( { :description => training_dataset.title, :filetype => "owl-dl", - :url => model.trainingDataset} ) if training_dataset + :url => training_dataset.uri} ) if training_dataset val_datasets.each do |data_uri| begin diff --git a/reach_reports/reach_test.rb b/reach_reports/reach_test.rb old mode 100644 new mode 100755 index 4a18f6d..13625e5 --- a/reach_reports/reach_test.rb +++ b/reach_reports/reach_test.rb @@ -7,7 +7,7 @@ require 'rack/test' require 'lib/test_util.rb' require 'test/test_examples.rb' -LOGGER = MyLogger.new(STDOUT) +LOGGER = OTLogger.new(STDOUT) LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S " LOGGER.formatter = Logger::Formatter.new @@ -115,7 +115,7 @@ class ReachTest < Test::Unit::TestCase # puts last_response.body #model_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/model/173393" - model_uri = "http://localhost/model/6" + model_uri = "http://localhost/model/1" #http://localhost/majority/class/model/15 #model_uri = "http://localhost/majority/class/model/15" # model_uri = "http://localhost/majority/class/model/91" @@ -141,15 +141,15 @@ class ReachTest < Test::Unit::TestCase # puts "RDF" # puts last_response.body - get '/reach_report/qmrf/'+id,nil,'HTTP_ACCEPT' => "application/qmrf-xml" - puts "XML" - puts last_response.body + #get '/reach_report/qmrf/'+id,nil,'HTTP_ACCEPT' => "application/qmrf-xml" + #puts "XML" + #puts last_response.body #r = ReachReports::QmrfReport.find_like( :QSAR_title => "Hamster") #puts r.collect{|rr| "report with id:"+rr.id.to_s}.inspect - File.new("/home/martin/tmp/qmr_rep_del_me.xml","w").puts last_response.body + #File.new("/home/martin/tmp/qmr_rep_del_me.xml","w").puts last_response.body #File.new("/home/martin/win/home/qmr_rep_del_me.xml","w").puts last_response.body #File.new("/home/martin/info_home/.public_html/qmr_rep_del_me.xml","w").puts last_response.body end diff --git a/report/environment.rb b/report/environment.rb old mode 100644 new mode 100755 index 1f62d0e..12e3272 --- a/report/environment.rb +++ b/report/environment.rb @@ -1,18 +1,19 @@ ['rubygems', 'logger', 'fileutils', 'sinatra', 'sinatra/url_for', 'rest_client', 'yaml', 'fileutils', 'mime/types', 'abbrev', 'rinruby', - 'rexml/document', 'ruby-plot', 'opentox-ruby-api-wrapper' ].each do |g| + 'rexml/document', 'ruby-plot', 'opentox-ruby' ].each do |g| require g end gem 'ruby-plot', '= 0.0.2' -R.quit +#R.quit module Reports end require "lib/ot_predictions.rb" -require "lib/active_record_setup.rb" +#require "lib/active_record_setup.rb" +require "lib/data_mapper_util.rb" require "report/plot_factory.rb" require "report/xml_report.rb" diff --git a/report/report_application.rb b/report/report_application.rb old mode 100644 new mode 100755 index a61a2df..6d6353f --- a/report/report_application.rb +++ b/report/report_application.rb @@ -123,11 +123,11 @@ delete '/report/:type/:id' do end post '/report/:type' do - task_uri = OpenTox::Task.as_task("Create report",url_for("/report/"+params[:type], :full), params) do |task| + task = OpenTox::Task.create("Create report",url_for("/report/"+params[:type], :full)) do |task| #,params perform do |rs| rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil,task) end end content_type "text/uri-list" - halt 202,task_uri+"\n" + halt 202,task.uri+"\n" end diff --git a/report/report_content.rb b/report/report_content.rb old mode 100644 new mode 100755 index eeb65e9..1345e6f --- a/report/report_content.rb +++ b/report/report_content.rb @@ -192,7 +192,7 @@ class Reports::ReportContent class_domain = validation_set.get_class_domain class_domain.size.times do |i| class_value = class_domain[i] - image_title = image_titles ? image_titles[i] : "ROC Plot for class-value '"+class_value+"'" + image_title = image_titles ? image_titles[i] : "ROC Plot for class-value '"+class_value.to_s+"'" image_caption = image_captions ? image_captions[i] : nil plot_file_name = "roc_plot"+@tmp_file_count.to_s+".svg" @tmp_file_count += 1 @@ -201,7 +201,7 @@ class Reports::ReportContent Reports::PlotFactory.create_roc_plot( plot_file_path, prediction_set, class_value, split_set_attribute, false )#prediction_set.size>1 ) @xml_report.add_imagefigure(section_roc, image_title, plot_file_name, "SVG", 120, image_caption) rescue RuntimeError => ex - msg = "WARNING could not create roc plot for class value '"+class_value+"': "+ex.message + msg = "WARNING could not create roc plot for class value '"+class_value.to_s+"': "+ex.message LOGGER.error(msg) rm_tmp_file(plot_file_name) @xml_report.add_paragraph(section_roc, msg) @@ -245,7 +245,7 @@ class Reports::ReportContent image_title = image_titles[i] else if class_value!=nil - image_title = rank_attribute.to_s+" Ranking Plot for class-value '"+class_value+"'" + image_title = rank_attribute.to_s+" Ranking Plot for class-value '"+class_value.to_s+"'" else image_title = rank_attribute.to_s+" Ranking Plot" end diff --git a/report/report_factory.rb b/report/report_factory.rb old mode 100644 new mode 100755 index ee66df7..11e9bfa --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -65,11 +65,12 @@ module Reports::ReportFactory report = Reports::ReportContent.new("Validation report") - if (val.classification?) + case val.feature_type + when "classification" report.add_result(validation_set, [:validation_uri] + VAL_ATTR_TRAIN_TEST + VAL_ATTR_CLASS, "Results", "Results") report.add_roc_plot(validation_set) report.add_confusion_matrix(val) - else #regression + when "regression" report.add_result(validation_set, [:validation_uri] + VAL_ATTR_TRAIN_TEST + VAL_ATTR_REGR, "Results", "Results") report.add_regression_plot(validation_set, :model_uri) end @@ -90,8 +91,8 @@ module Reports::ReportFactory raise Reports::BadRequest.new("num validations ("+validation_set.size.to_s+") is not equal to num folds ("+ validation_set.unique_value(:num_folds).to_s+")") unless validation_set.unique_value(:num_folds)==validation_set.size raise Reports::BadRequest.new("num different folds is not equal to num validations") unless validation_set.num_different_values(:crossvalidation_fold)==validation_set.size - raise Reports::BadRequest.new("validations must be either all regression, "+ - +"or all classification validations") unless validation_set.all_classification? or validation_set.all_regression? + raise Reports::BadRequest.new("validations must have unique feature type, i.e. must be either all regression, "+ + +"or all classification validations") unless validation_set.unique_feature_type pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) ) merged = validation_set.merge([:crossvalidation_id]) @@ -100,14 +101,15 @@ module Reports::ReportFactory #puts merged.get_values(:percent_correct_variance, false).inspect report = Reports::ReportContent.new("Crossvalidation report") - if (validation_set.all_classification?) + case validation_set.unique_feature_type + when "classification" report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold],"Mean Results","Mean Results") report.add_roc_plot(validation_set, nil, "ROC Plots over all folds") report.add_roc_plot(validation_set, :crossvalidation_fold) report.add_confusion_matrix(merged.validations[0]) report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_CLASS-[:num_folds], "Results","Results",nil,"validation") - else #regression + when "regression" report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold],"Mean Results","Mean Results") report.add_regression_plot(validation_set, :crossvalidation_fold) report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_REGR-[:num_folds], "Results","Results") @@ -124,8 +126,8 @@ module Reports::ReportFactory #validation_set.to_array([:test_dataset_uri, :model_uri, :algorithm_uri], false).each{|a| puts a.inspect} raise Reports::BadRequest.new("num validations is not >1") unless validation_set.size>1 - raise Reports::BadRequest.new("validations must be either all regression, "+ - "or all classification validations") unless validation_set.all_classification? or validation_set.all_regression? + raise Reports::BadRequest.new("validations must have unique feature type, i.e. must be either all regression, "+ + +"or all classification validations") unless validation_set.unique_feature_type raise Reports::BadRequest.new("number of different algorithms <2: "+ validation_set.get_values(:algorithm_uri).inspect) if validation_set.num_different_values(:algorithm_uri)<2 @@ -168,7 +170,8 @@ module Reports::ReportFactory end - if (validation_set.all_classification?) + case validation_set.unique_feature_type + when "classification" attributes = VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold] attributes = ([ :dataset_uri ] + attributes).uniq @@ -190,7 +193,7 @@ module Reports::ReportFactory report.end_section end - else # regression + when "regression" raise Reports::BadRequest.new("algorithm comparison for regression not yet implemented") end task.progress(100) if task diff --git a/report/report_persistance.rb b/report/report_persistance.rb old mode 100644 new mode 100755 index 1bd37b0..d24c792 --- a/report/report_persistance.rb +++ b/report/report_persistance.rb @@ -181,14 +181,28 @@ end module Reports - class ReportData < ActiveRecord::Base + #class ReportData < ActiveRecord::Base +# serialize :validation_uris +# serialize :crossvalidation_uris +# serialize :algorithm_uris +# serialize :model_uris +# alias_attribute :date, :created_at + + class ReportData + include DataMapper::Resource - serialize :validation_uris - serialize :crossvalidation_uris - serialize :algorithm_uris - serialize :model_uris + property :id, Serial + property :report_uri, String, :length => 255 + property :report_type, String, :length => 255 + property :created_at, DateTime + property :validation_uris, Object + property :crossvalidation_uris, Object + property :model_uris, Object + property :algorithm_uris, Object - alias_attribute :date, :created_at + def date + created_at + end def get_content_as_hash map = {} @@ -216,23 +230,40 @@ module Reports raise "report meta data missing" unless meta_data report = ReportData.new(meta_data) report.save #to set id - report.attributes = { :report_type => type, :report_uri => uri_provider.get_uri(type, report.id) } - report.save +# report.attributes = { :report_type => type, :report_uri => uri_provider.get_uri(type, report.id) } +# report.save + report.update :report_type => type, :report_uri => uri_provider.get_uri(type, report.id) new_report_with_id(report_content, type, report.id) end def list_reports(type, filter_params={}) filter_params["report_type"]=type unless filter_params.has_key?("report_type") - ReportData.find_like(filter_params).delete_if{|r| r.report_type!=type}.collect{ |r| r.id } + #ReportData.find_like(filter_params).delete_if{|r| r.report_type!=type}.collect{ |r| r.id } + + filter_params = Lib::DataMapperUtil.check_params(ReportData, filter_params) + # unfortunately, datamapper does not allow searching in Objects + # do filtering for list = Object params manually + list_params = {} + [:validation_uris, :crossvalidation_uris, :algorithm_uris, :model_uris].each do |l| + list_params[l] = filter_params.delete(l) if filter_params.has_key?(l) + end + + reports = ReportData.all(filter_params).delete_if{|r| r.report_type!=type} + list_params.each do |k,v| + reports = reports.delete_if{ |r| !r.send(k).include?(v) } + end + reports.collect{ |r| r.id } end def get_report(type, id, format, force_formating, params) - begin - report = ReportData.find(:first, :conditions => {:id => id, :report_type => type}) - rescue ActiveRecord::RecordNotFound - raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") - end + report = ReportData.first({:id => id, :report_type => type}) + raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") unless report +# begin +# report = ReportData.find(:first, :conditions => {:id => id, :report_type => type}) +# rescue ActiveRecord::RecordNotFound +# raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") +# end case format when "application/rdf+xml" @@ -245,13 +276,51 @@ module Reports end def delete_report(type, id) - begin - report = ReportData.find(:first, :conditions => {:id => id, :report_type => type}) - rescue ActiveRecord::RecordNotFound - raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") - end - ReportData.delete(id) +# begin +# report = ReportData.find(:first, :conditions => {:id => id, :report_type => type}) +# rescue ActiveRecord::RecordNotFound +# raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") +# end +# ReportData.delete(id) + report = ReportData.first({:id => id, :report_type => type}) + raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") unless report + report.destroy super end end end + +Reports::ReportData.auto_upgrade! +Reports::ReportData.raise_on_save_failure = true + +#module Reports +# def self.check_filter_params(model, filter_params) +# prop_names = model.properties.collect{|p| p.name.to_s} +# filter_params.keys.each do |k| +# key = k.to_s +# unless prop_names.include?(key) +# key = key.from_rdf_format +# unless prop_names.include?(key) +# key = key+"_uri" +# unless prop_names.include?(key) +# key = key+"s" +# unless prop_names.include?(key) +# err = "no attribute found: '"+k.to_s+"'" +# if $sinatra +# $sinatra.halt 400,err +# else +# raise err +# end +# end +# end +# end +# end +# filter_params[key] = filter_params.delete(k) +# end +# filter_params +# end +# +# def ReportData.all( params ) +# super Reports.check_filter_params( ReportData, params ) +# end +#end diff --git a/report/report_test.rb b/report/report_test.rb old mode 100644 new mode 100755 index 5351c7d..f579802 --- a/report/report_test.rb +++ b/report/report_test.rb @@ -33,7 +33,10 @@ class Reports::ApplicationTest < Test::Unit::TestCase #puts uri #get uri - get '/report/validation/117',nil,'HTTP_ACCEPT' => "text/html" + get '/report/crossvalidation',:model=>"http://localhost/majority/class/model/101" + puts last_response.body.to_s + + #get '/report/validation/117',nil,'HTTP_ACCEPT' => "text/html" #post '/report/validation/1/format_html',:css_style_sheet=>"http://apps.ideaconsult.net:8180/ToxPredict/style/global.css" #post 'http://ot.validation.de/report/validation',:validation_uris=>"http://ot.validation.de/1" @@ -119,12 +122,12 @@ end #class Reports::ReportServiceTest < Test::Unit::TestCase # include Lib::TestUtil # -# WS_VAL = @@config[:services]["opentox-validation"] -# WS_DATA=@@config[:services]["opentox-dataset"] +# WS_VAL = CONFIG[:services]["opentox-validation"] +# WS_DATA=CONFIG[:services]["opentox-dataset"] # FILE=File.new("data/hamster_carcinogenicity.owl","r") # -# WS_CLASS_ALG=File.join(@@config[:services]["opentox-algorithm"],"lazar") -# WS_FEATURE_ALG=File.join(@@config[:services]["opentox-algorithm"],"fminer") +# WS_CLASS_ALG=File.join(CONFIG[:services]["opentox-algorithm"],"lazar") +# WS_FEATURE_ALG=File.join(CONFIG[:services]["opentox-algorithm"],"fminer") # # #WS_CLASS_ALG_2="localhost:4008/algorithm" # #WS_FEATURE_ALG_2=nil diff --git a/report/validation_access.rb b/report/validation_access.rb old mode 100644 new mode 100755 index a0b6355..170cdfd --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -36,8 +36,8 @@ class Reports::ValidationAccess raise "not implemented" end - # is validation classification? - def classification?(validation) + # is validation classification/regression? + def feature_type(validation) raise "not implemented" end @@ -49,9 +49,9 @@ end class Reports::ValidationDB < Reports::ValidationAccess - def initialize() - @model_store = {} - end +# def initialize() +# @model_store = {} +# end def resolve_cv_uris(validation_uris) res = [] @@ -60,13 +60,15 @@ class Reports::ValidationDB < Reports::ValidationAccess cv_id = u.split("/")[-1].to_i cv = nil begin - cv = Lib::Crossvalidation.find( cv_id ) + #cv = Lib::Crossvalidation.find( cv_id ) + cv = Lib::Crossvalidation.get( cv_id ) rescue => ex raise "could not access crossvalidation with id "+validation_id.to_s+", error-msg: "+ex.message end raise Reports::BadRequest.new("crossvalidation with id '"+cv_id.to_s+"' not found") unless cv raise Reports::BadRequest.new("crossvalidation with id '"+cv_id.to_s+"' not finished") unless cv.finished - res += Lib::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s} + #res += Lib::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s} + res += Lib::Validation.all( :crossvalidation_id => cv_id ).collect{|v| v.validation_uri.to_s } else res += [u.to_s] end @@ -82,7 +84,8 @@ class Reports::ValidationDB < Reports::ValidationAccess (validation_id.to_i > 0 || validation_id.to_s=="0" ) v = nil begin - v = Lib::Validation.find(validation_id) + #v = Lib::Validation.find(validation_id) + v = Lib::Validation.get(validation_id) rescue => ex raise "could not access validation with id "+validation_id.to_s+", error-msg: "+ex.message end @@ -102,7 +105,8 @@ class Reports::ValidationDB < Reports::ValidationAccess def init_cv(validation) - cv = Lib::Crossvalidation.find(validation.crossvalidation_id) + #cv = Lib::Crossvalidation.find(validation.crossvalidation_id) + cv = Lib::Crossvalidation.get(validation.crossvalidation_id) raise Reports::BadRequest.new "no crossvalidation found with id "+validation.crossvalidation_id.to_s unless cv Lib::CROSS_VAL_PROPS.each do |p| @@ -111,35 +115,39 @@ class Reports::ValidationDB < Reports::ValidationAccess end def get_predictions(validation, task=nil) - Lib::OTPredictions.new( validation.classification?, validation.test_dataset_uri, + Lib::OTPredictions.new( validation.feature_type, validation.test_dataset_uri, validation.test_target_dataset_uri, validation.prediction_feature, validation.prediction_dataset_uri, validation.predicted_variable, task) end def get_class_domain( validation ) - OpenTox::Feature.domain( validation.prediction_feature ) + OpenTox::Feature.new( validation.prediction_feature ).domain end - def classification?( validation ) - get_model(validation).classification? + def feature_type( validation ) + OpenTox::Model::Generic.new(validation.model_uri).feature_type + #get_model(validation).classification? end def predicted_variable(validation) - get_model(validation).predictedVariables - end - - private - def get_model(validation) raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) - model = @model_store[validation.model_uri] - unless model - model = OpenTox::Model::PredictionModel.find(validation.model_uri) - raise "model not found '"+validation.model_uri+"'" unless validation.model_uri && model - @model_store[validation.model_uri] = model - end - return model + model = OpenTox::Model::Generic.find(validation.model_uri) + raise Reports::NotFound.new "model not found '"+validation.model_uri+"'" unless model + model.metadata[OT.predictedVariables] + #get_model(validation).predictedVariables end +# private +# def get_model(validation) +# raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) +# model = @model_store[validation.model_uri] +# unless model +# model = OpenTox::Model::PredictionModel.find(validation.model_uri) +# raise "model not found '"+validation.model_uri+"'" unless validation.model_uri && model +# @model_store[validation.model_uri] = model +# end +# return model +# end end diff --git a/report/validation_data.rb b/report/validation_data.rb old mode 100644 new mode 100755 index d558c88..c4eed12 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -107,11 +107,11 @@ module Reports @class_domain end - # is classification validation? cache to save resr-calls + # is classification/regression validation? cache to save rest-calls # - def classification? - return @is_classification if @is_classification!=nil - @is_classification = @@validation_access.classification?(self) + def feature_type + return @feature_type if @feature_type!=nil + @feature_type = @@validation_access.feature_type(self) end def predicted_variable @@ -130,7 +130,7 @@ module Reports def validation_report_uri #puts "searching for validation report: "+self.validation_uri.to_s return @validation_report_uri if @validation_report_uri!=nil - ids = @@persistance.list_reports("validation",{:validation=>validation_uri }) + ids = @@persistance.list_reports("validation",{:validation_uris=>validation_uri }) @validation_report_uri = Reports::ReportService.instance.get_uri("validation",ids[-1]) if ids and ids.size>0 end @@ -259,19 +259,12 @@ module Reports end end - # checks weather all validations are classification validations + # checks weather all validations are classification/regression validations # - def all_classification? - return unique_value("classification?") + def unique_feature_type + return unique_value("feature_type") end - # checks weather all validations are regression validations - # - def all_regression? - # WARNING, NOT TRUE: !all_classification == all_regression? - return unique_value("classification?")==false - end - # returns a new set with all validation that have values as specified in the map # # call-seq: @@ -348,39 +341,42 @@ module Reports array.push(attributes.collect{|a| a.to_s.nice_attr}) attribute_not_nil = Array.new(attributes.size) @validations.each do |v| - index = 0 + index = -1 array.push(attributes.collect do |a| + index += 1 if VAL_ATTR_VARIANCE.index(a) variance = v.send( (a.to_s+"_variance").to_sym ) end #variance = " +- "+variance.to_nice_s if variance - attribute_not_nil[index] = true if remove_nil_attributes and v.send(a)!=nil - index += 1 val = v.send(a) - - class_domain = get_domain_for_attr(a) - # get domain for classification attribute, i.e. ["true","false"] - if class_domain.size==1 && class_domain[0]!=nil - # or the attribute has a complementary value, i.e. true_positive_rate - # -> domain is reduced to one class value - raise "illegal state" unless (val.is_a?(Hash)) - val = val[class_domain[0]] - end - - if variance - if (val.is_a?(Array)) - raise "not implemented" - elsif (val.is_a?(Hash)) - val.collect{ |i,j| i.to_nice_s+": "+j.to_nice_s + " +- " + - variance[i].to_nice_s }.join(", ") + if val==nil || val.to_s.chomp.size==0 + '' + else + attribute_not_nil[index] = true if remove_nil_attributes + + class_domain = get_domain_for_attr(a) + # get domain for classification attribute, i.e. ["true","false"] + if class_domain.size==1 && class_domain[0]!=nil + # or the attribute has a complementary value, i.e. true_positive_rate + # -> domain is reduced to one class value + raise "illegal state, value for "+a.to_s+" is no hash: '"+val.to_s+"'" unless (val.is_a?(Hash)) + val = val[class_domain[0]] + end + + if variance + if (val.is_a?(Array)) + raise "not implemented" + elsif (val.is_a?(Hash)) + val.collect{ |i,j| i.to_nice_s+": "+j.to_nice_s + " +- " + + variance[i].to_nice_s }.join(", ") + else + val.to_nice_s + " +- " + variance.to_nice_s + end else - val.to_nice_s + " +- " + variance.to_nice_s + val.to_nice_s end - else - val.to_nice_s end - end) end diff --git a/report/xml_report.rb b/report/xml_report.rb old mode 100644 new mode 100755 index 76844b3..46f2a67 --- a/report/xml_report.rb +++ b/report/xml_report.rb @@ -184,7 +184,7 @@ module Reports row = Element.new("row") r.each do |v| entry = Element.new("entry") - if auto_link_urls && v.to_s =~ /depict/ #PENDING + if auto_link_urls && v.to_s =~ /depict/ || v.to_s =~ /image$/ #PENDING add_image(entry, v.to_s) elsif auto_link_urls && v.to_s =~ /^http:\/\// add_url(entry, v.to_s, v.to_s) @@ -241,7 +241,7 @@ module Reports end end - @doc.write(out,2, true, true) + @doc.write(out) #,2, true, true) out.flush end diff --git a/report/xml_report_util.rb b/report/xml_report_util.rb old mode 100644 new mode 100755 index fcb7d96..fd1a179 --- a/report/xml_report_util.rb +++ b/report/xml_report_util.rb @@ -35,21 +35,22 @@ module Reports end confusion = [] - confusion.push( [ "", "", "actual" ] + [""] * num_classes ) - confusion.push( [ "", "" ] + class_values + [ "total"]) + confusion.push( [ " ", " ", "actual" ] + [" "] * num_classes ) + confusion.push( [ " ", " " ] + class_values + [ "total"]) class_values.each do |predicted| - row = [ (confusion.size==2 ? "predicted" : ""), predicted ] + row = [ (confusion.size==2 ? "predicted" : " "), predicted ] class_values.each do |actual| row.push( confusion_matrix[{:confusion_matrix_actual => actual, :confusion_matrix_predicted => predicted}].to_nice_s ) end row.push( sum_predicted[predicted].to_nice_s ) confusion.push( row ) end - last_row = [ "", "total" ] + last_row = [ " ", "total" ] class_values.each do |actual| last_row.push( sum_actual[actual].to_nice_s ) end + last_row.push(" ") confusion.push( last_row ) return confusion diff --git a/test/test_application.rb b/test/test_application.rb old mode 100644 new mode 100755 index 48cced7..c6c7963 --- a/test/test_application.rb +++ b/test/test_application.rb @@ -2,7 +2,7 @@ post '/test_validation/?' do validationExamples = ValidationExamples.select(params[:select]) return "please \"select\" a single validation example:\n"+ValidationExamples.list if validationExamples.size!=1 or validationExamples[0].size!=1 - OpenTox::Task.as_task("Test validation",url_for("/test_validation",:full), params) do + task = OpenTox::Task.create("Test validation",url_for("/test_validation",:full)) do #,params v = validationExamples[0][0] ex = v.new ex.upload_files @@ -13,5 +13,6 @@ post '/test_validation/?' do raise ex.report_error if ex.report_error ex.validation_uri + (params[:report] ? ","+ex.report_uri : "") end + task.uri end diff --git a/test/test_examples.rb b/test/test_examples.rb old mode 100644 new mode 100755 index 8927fb1..63fb505 --- a/test/test_examples.rb +++ b/test/test_examples.rb @@ -19,15 +19,15 @@ module ValidationExamples class MajorityIrisCrossvalidation < IrisCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") super end end class LazarIrisCrossvalidation < IrisCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer") super end end @@ -43,15 +43,65 @@ module ValidationExamples class LazarIrisSplit < IrisSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer") super end end class MajorityIrisSplit < IrisSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + ######################################################################################################## + + class EPAFHMSplit < SplitTestValidation + def initialize + @dataset_file = File.new("data/EPAFHM.csv","r") + #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" + @split_ratio = 0.95 + end + end + + class LazarEPAFHMSplit < EPAFHMSplit + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + super + end + end + + class MajorityEPAFHMSplit < EPAFHMSplit + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + ######################################################################################################## + + class EPAFHMCrossvalidation < CrossValidation + def initialize + @dataset_file = File.new("data/EPAFHM.csv","r") + #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" + @num_folds = 10 + end + end + + class MajorityEPAFHMCrossvalidation < EPAFHMCrossvalidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + class LazarEPAFHMCrossvalidation < EPAFHMCrossvalidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end @@ -61,21 +111,22 @@ module ValidationExamples class HamsterSplit < SplitTestValidation def initialize @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity" end end class LazarHamsterSplit < HamsterSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end class MajorityHamsterSplit < HamsterSplit def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end @@ -85,21 +136,22 @@ module ValidationExamples class HamsterBootstrapping < BootstrappingValidation def initialize @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity" end end class LazarHamsterBootstrapping < HamsterBootstrapping def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end class MajorityHamsterBootstrapping < HamsterBootstrapping def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end @@ -111,21 +163,22 @@ module ValidationExamples @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r") @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity" end end class MajorityHamsterTrainingTest < HamsterTrainingTest def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end class LazarHamsterTrainingTest < HamsterTrainingTest def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end @@ -135,22 +188,23 @@ module ValidationExamples class HamsterCrossvalidation < CrossValidation def initialize @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") - @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity" @num_folds = 10 end end class MajorityHamsterCrossvalidation < HamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end class LazarHamsterCrossvalidation < HamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") super end end @@ -166,15 +220,15 @@ module ValidationExamples class MajorityISTHamsterCrossvalidation < ISTHamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end class LazarISTHamsterCrossvalidation < ISTHamsterCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") - @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer") super end end @@ -215,7 +269,7 @@ module ValidationExamples class MajorityISTRatLiverCrossvalidation < ISTRatLiverCrossvalidation def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm") super end end @@ -283,7 +337,7 @@ module ValidationExamples class MajorityCacoTrainingTest < CacoTrainingTest def initialize - @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") super end end @@ -331,6 +385,12 @@ module ValidationExamples "12" => [ LazarHamsterBootstrapping, MajorityHamsterBootstrapping ], "12a" => [ LazarHamsterBootstrapping ], "12b" => [ MajorityHamsterBootstrapping ], + + "13a" => [ LazarEPAFHMSplit ], + "13b" => [ MajorityEPAFHMSplit ], + + "14a" => [ LazarEPAFHMCrossvalidation ], + "14b" => [ MajorityEPAFHMCrossvalidation ], } def self.list diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb old mode 100644 new mode 100755 index cc31fe5..b9a1c99 --- a/test/test_examples_util.rb +++ b/test/test_examples_util.rb @@ -4,20 +4,40 @@ module ValidationExamples class Util @@dataset_uris = {} + @@prediction_features = {} - def self.upload_dataset(file, dataset_service=@@config[:services]["opentox-dataset"], file_type="application/x-yaml") + def self.upload_dataset(file, dataset_service=CONFIG[:services]["opentox-dataset"]) #, file_type="application/x-yaml") raise "File not found: "+file.path.to_s unless File.exist?(file.path) if @@dataset_uris[file.path.to_s]==nil - data = File.read(file.path) - data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp - @@dataset_uris[file.path.to_s] = data_uri - LOGGER.debug "uploaded dataset: "+data_uri + LOGGER.debug "uploading file: "+file.path.to_s + if (file.path =~ /yaml$/) + data = File.read(file.path) + #data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp + #@@dataset_uris[file.path.to_s] = data_uri + #LOGGER.debug "uploaded dataset: "+data_uri + d = OpenTox::Dataset.create + d.load_yaml(data) + d.save + @@dataset_uris[file.path.to_s] = d.uri + elsif (file.path =~ /csv$/) + d = OpenTox::Dataset.create_from_csv_file(file.path) + raise "num features not 1 (="+d.features.keys.size.to_s+"), what to predict??" if d.features.keys.size != 1 + @@prediction_features[file.path.to_s] = d.features.keys[0] + @@dataset_uris[file.path.to_s] = d.uri + else + raise "unknown file type: "+file.path.to_s + end + LOGGER.debug "uploaded dataset: "+d.uri else LOGGER.debug "file already uploaded: "+@@dataset_uris[file.path.to_s] end return @@dataset_uris[file.path.to_s] end + def self.prediction_feature_for_file(file) + @@prediction_features[file.path.to_s] + end + def self.build_compare_report(validation_examples) @comp = validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri @@ -35,12 +55,12 @@ module ValidationExamples end def self.validation_post(uri, params) + if $test_case - #puts "posting: "+uri+","+params.inspect - $test_case.post uri,params + $test_case.post uri,params return wait($test_case.last_response.body) else - return OpenTox::RestClientWrapper.post(File.join(@@config[:services]["opentox-validation"],uri),params).to_s + return OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],uri),params).to_s end end @@ -50,16 +70,27 @@ module ValidationExamples $test_case.get uri,nil,'HTTP_ACCEPT' => accept_header return wait($test_case.last_response.body) else - return OpenTox::RestClientWrapper.get(File.join(@@config[:services]["opentox-validation"],uri),{:accept => accept_header}) + return OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"],uri),{:accept => accept_header}) + end + end + + def self.validation_delete(uri, accept_header='application/rdf+xml') + + if $test_case + $test_case.delete uri,nil,'HTTP_ACCEPT' => accept_header + return wait($test_case.last_response.body) + else + return OpenTox::RestClientWrapper.delete(File.join(CONFIG[:services]["opentox-validation"],uri),{:accept => accept_header}) end end + def self.wait(uri) - if OpenTox::Utils.task_uri?(uri) - task = OpenTox::Task.find(uri) + if uri.task_uri? + task = OpenTox::Task.find(uri.to_s.chomp) task.wait_for_completion - raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error? - uri = task.resultURI + raise "task failed: "+uri.to_s+", error is:\n"+task.description.to_s if task.error? + uri = task.result_uri end uri end @@ -214,6 +245,10 @@ module ValidationExamples send("#{uri.to_s}=".to_sym, Util.upload_dataset(send(file))) end end + + if (params.include?(:prediction_feature) and @prediction_feature==nil and @dataset_uri and @dataset_file) + @prediction_feature = Util.prediction_feature_for_file(@dataset_file) + end end def check_requirements @@ -222,21 +257,42 @@ module ValidationExamples end end + def delete + begin + if @validation_uri =~ /crossvalidation/ + cv = "crossvalidation/" + else + cv = "" + end + Util.validation_delete '/'+cv+@validation_uri.split('/')[-1] if @validation_uri + rescue => ex + puts "Could not delete validation: "+ex.message + end + begin + Util.validation_delete '/report/'+report_type+'/'+@report_uri.split('/')[-1] if @report_uri + rescue => ex + puts "Could not delete report:' "+@report_uri+" "+ex.message + end + end + def report begin @report_uri = Util.validation_post '/report/'+report_type,{:validation_uris => @validation_uri} rescue => ex + puts "could not create report: "+ex.message + raise ex @report_error = ex.message end end def validate - #begin + begin @validation_uri = Util.validation_post '/'+validation_type, get_params - #rescue => ex - # @validation_error = ex.message - # LOGGER.error ex.message - #end + rescue => ex + puts "could not validate: "+ex.message + @validation_error = ex.message + LOGGER.error ex.message + end end def compare_yaml_vs_rdf @@ -308,6 +364,10 @@ module ValidationExamples def opt_params [ :algorithm_params, :test_target_dataset_uri ] end + + def validation_type + "training_test_validation" + end end class SplitTestValidation < ValidationExample diff --git a/validation/validation_application.rb b/validation/validation_application.rb old mode 100644 new mode 100755 index 2a2bcb0..9c6c60c --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -1,16 +1,17 @@ -[ 'rubygems', 'sinatra', 'sinatra/url_for', 'opentox-ruby-api-wrapper' ].each do |lib| +[ 'rubygems', 'sinatra', 'sinatra/url_for', 'opentox-ruby' ].each do |lib| require lib end require 'lib/merge.rb' -require 'lib/active_record_setup.rb' +#require 'lib/active_record_setup.rb' require 'validation/validation_service.rb' get '/crossvalidation/?' do LOGGER.info "list all crossvalidations" - uri_list = Validation::Crossvalidation.find_like(params).collect{ |cv| cv.crossvalidation_uri }.join("\n")+"\n" + uri_list = Validation::Crossvalidation.all.collect{ |cv| cv.crossvalidation_uri }.join("\n")+"\n" + #uri_list = Validation::Crossvalidation.find_like(params).collect{ |cv| cv.crossvalidation_uri }.join("\n")+"\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "Single validations: "+$sinatra.url_for("/",:full)+"\n"+ @@ -28,8 +29,7 @@ get '/crossvalidation/?' do end post '/crossvalidation/?' do - content_type "text/uri-list" - task_uri = OpenTox::Task.as_task( "Perform crossvalidation", url_for("/crossvalidation", :full), params ) do |task| + task = OpenTox::Task.create( "Perform crossvalidation", url_for("/crossvalidation", :full) ) do |task| #, params LOGGER.info "creating crossvalidation "+params.inspect halt 400, "dataset_uri missing" unless params[:dataset_uri] halt 400, "algorithm_uri missing" unless params[:algorithm_uri] @@ -44,20 +44,25 @@ post '/crossvalidation/?' do cv.perform_cv( params[:prediction_feature], params[:algorithm_params], task ) cv.crossvalidation_uri end - halt 202,task_uri+"\n" + content_type 'text/uri-list' + halt 503,task.uri+"\n" if task.status == "Cancelled" + halt 202,task.uri end post '/crossvalidation/cleanup/?' do LOGGER.info "crossvalidation cleanup, starting..." content_type "text/uri-list" deleted = [] - Validation::Crossvalidation.find_like(params).each do |cv| - num_vals = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv.id } ).size - if cv.num_folds != num_vals - LOGGER.debug "delete cv with id:"+cv.id.to_s+" num-folds should be "+cv.num_folds.to_s+", is "+num_vals.to_s + #Validation::Crossvalidation.find_like(params).each do |cv| + Validation::Crossvalidation.all( { :finished => false } ).each do |cv| + #num_vals = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv.id } ).size + #num_vals = Validation::Validation.all( :crossvalidation_id => cv.id ).size + #if cv.num_folds != num_vals || !cv.finished + LOGGER.debug "delete cv with id:"+cv.id.to_s+", finished is false" deleted << cv.crossvalidation_uri - Validation::Crossvalidation.delete(cv.id) - end + #Validation::Crossvalidation.delete(cv.id) + cv.delete + #end end LOGGER.info "crossvalidation cleanup, deleted "+deleted.size.to_s+" cvs" deleted.join("\n")+"\n" @@ -73,11 +78,13 @@ end get '/crossvalidation/:id' do LOGGER.info "get crossvalidation with id "+params[:id].to_s - begin - crossvalidation = Validation::Crossvalidation.find(params[:id]) - rescue ActiveRecord::RecordNotFound => ex - halt 404, "Crossvalidation '#{params[:id]}' not found." - end +# begin +# #crossvalidation = Validation::Crossvalidation.find(params[:id]) +# rescue ActiveRecord::RecordNotFound => ex +# halt 404, "Crossvalidation '#{params[:id]}' not found." +# end + crossvalidation = Validation::Crossvalidation.get(params[:id]) + halt 404,"Crossvalidation '#{params[:id]}' not found." unless crossvalidation case request.env['HTTP_ACCEPT'].to_s when "application/rdf+xml" @@ -104,19 +111,24 @@ end get '/crossvalidation/:id/statistics' do LOGGER.info "get merged validation-result for crossvalidation with id "+params[:id].to_s - begin - crossvalidation = Validation::Crossvalidation.find(params[:id]) - rescue ActiveRecord::RecordNotFound => ex - halt 404, "Crossvalidation '#{params[:id]}' not found." - end +# begin + #crossvalidation = Validation::Crossvalidation.find(params[:id]) +# rescue ActiveRecord::RecordNotFound => ex +# halt 404, "Crossvalidation '#{params[:id]}' not found." +# end + #crossvalidation = Validation::Crossvalidation.find(params[:id]) + crossvalidation = Validation::Crossvalidation.get(params[:id]) + + halt 404,"Crossvalidation '#{params[:id]}' not found." unless crossvalidation halt 400,"Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished Lib::MergeObjects.register_merge_attributes( Validation::Validation, - Lib::VAL_MERGE_AVG,Lib::VAL_MERGE_SUM,Lib::VAL_MERGE_GENERAL-[:validation_uri,:crossvalidation_uri]) unless + Lib::VAL_MERGE_AVG,Lib::VAL_MERGE_SUM,Lib::VAL_MERGE_GENERAL-[:date,:validation_uri,:crossvalidation_uri]) unless Lib::MergeObjects.merge_attributes_registered?(Validation::Validation) - v = Lib::MergeObjects.merge_array_objects( Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) ) - v.date = nil + #v = Lib::MergeObjects.merge_array_objects( Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) ) + v = Lib::MergeObjects.merge_array_objects( Validation::Validation.all( :crossvalidation_id => params[:id] ) ) + v.created_at = nil v.id = nil case request.env['HTTP_ACCEPT'].to_s @@ -136,12 +148,16 @@ end delete '/crossvalidation/:id/?' do LOGGER.info "delete crossvalidation with id "+params[:id].to_s content_type "text/plain" - begin - crossvalidation = Validation::Crossvalidation.find(params[:id]) - rescue ActiveRecord::RecordNotFound => ex - halt 404, "Crossvalidation '#{params[:id]}' not found." - end - Validation::Crossvalidation.delete(params[:id]) +# begin + #crossvalidation = Validation::Crossvalidation.find(params[:id]) +# rescue ActiveRecord::RecordNotFound => ex +# halt 404, "Crossvalidation '#{params[:id]}' not found." +# end +# Validation::Crossvalidation.delete(params[:id]) + + cv = Validation::Crossvalidation.get(params[:id]) + halt 404,"Crossvalidation '#{params[:id]}' not found." unless cv + cv.delete end #get '/crossvalidation/:id/validations' do @@ -158,14 +174,16 @@ end get '/crossvalidation/:id/predictions' do LOGGER.info "get predictions for crossvalidation with id "+params[:id].to_s begin - crossvalidation = Validation::Crossvalidation.find(params[:id]) + #crossvalidation = Validation::Crossvalidation.find(params[:id]) + crossvalidation = Validation::Crossvalidation.get(params[:id]) rescue ActiveRecord::RecordNotFound => ex halt 404, "Crossvalidation '#{params[:id]}' not found." end halt 400,"Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished content_type "application/x-yaml" - validations = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) + #validations = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) + validations = Validation::Validation.all( :crossvalidation_id => params[:id] ) p = Lib::OTPredictions.to_array( validations.collect{ |v| v.compute_validation_stats_with_model(nil, true) } ).to_yaml case request.env['HTTP_ACCEPT'].to_s @@ -184,8 +202,11 @@ get '/crossvalidation/:id/predictions' do end get '/?' do + LOGGER.info "list all validations, params: "+params.inspect - uri_list = Validation::Validation.find_like(params).collect{ |v| v.validation_uri }.join("\n")+"\n" + #uri_list = Validation::Validation.find_like(params).collect{ |v| v.validation_uri }.join("\n")+"\n" + uri_list = Validation::Validation.all.collect{ |v| v.validation_uri }.join("\n")+"\n" + if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "To perform a validation:\n"+ @@ -216,7 +237,7 @@ post '/test_set_validation' do content_type "text/uri-list" LOGGER.info "creating test-set-validation "+params.inspect if params[:model_uri] and params[:test_dataset_uri] and !params[:training_dataset_uri] and !params[:algorithm_uri] - task_uri = OpenTox::Task.as_task( "Perform test-set-validation", url_for("/", :full), params ) do |task| + task = OpenTox::Task.create( "Perform test-set-validation", url_for("/", :full) ) do |task| #, params v = Validation::Validation.new :validation_type => "test_set_validation", :model_uri => params[:model_uri], :test_dataset_uri => params[:test_dataset_uri], @@ -225,7 +246,7 @@ post '/test_set_validation' do v.validate_model( task ) v.validation_uri end - halt 202,task_uri+"\n" + halt 202,task.uri+"\n" else halt 400, "illegal parameters, pls specify model_uri and test_dataset_uri\n"+ "params given: "+params.inspect @@ -234,7 +255,10 @@ end get '/test_set_validation' do LOGGER.info "list all test-set-validations, params: "+params.inspect - uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "test_set_validation" } ).collect{ |v| v.validation_uri }.join("\n")+"\n" + + #uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "test_set_validation" } ).collect{ |v| v.validation_uri }.join("\n")+"\n" + uri_list = Validation::Validation.all( :validation_type => "test_set_validation" ).collect{ |v| v.validation_uri }.join("\n")+"\n" + if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "All validations: "+$sinatra.url_for("/",:full)+"\n"+ @@ -252,10 +276,9 @@ get '/test_set_validation' do end post '/training_test_validation/?' do - content_type "text/uri-list" LOGGER.info "creating training-test-validation "+params.inspect if params[:algorithm_uri] and params[:training_dataset_uri] and params[:test_dataset_uri] and params[:prediction_feature] and !params[:model_uri] - task_uri = OpenTox::Task.as_task( "Perform training-test-validation", url_for("/", :full), params ) do |task| + task = OpenTox::Task.create( "Perform training-test-validation", url_for("/", :full) ) do |task| #, params v = Validation::Validation.new :validation_type => "training_test_validation", :algorithm_uri => params[:algorithm_uri], :training_dataset_uri => params[:training_dataset_uri], @@ -265,7 +288,8 @@ post '/training_test_validation/?' do v.validate_algorithm( params[:algorithm_params], task ) v.validation_uri end - halt 202,task_uri+"\n" + content_type "text/uri-list" + halt 202,task.uri+"\n" else halt 400, "illegal parameters, pls specify algorithm_uri, training_dataset_uri, test_dataset_uri, prediction_feature\n"+ "params given: "+params.inspect @@ -274,7 +298,8 @@ end get '/training_test_validation' do LOGGER.info "list all training-test-validations, params: "+params.inspect - uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "training_test_validation" } ).collect{ |v| v.validation_uri }.join("\n")+"\n" + #uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "training_test_validation" } ).collect{ |v| v.validation_uri }.join("\n")+"\n" + uri_list = Validation::Validation.all( :validation_type => "training_test_validation" ).collect{ |v| v.validation_uri }.join("\n")+"\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "All validations: "+$sinatra.url_for("/",:full)+"\n"+ @@ -298,7 +323,7 @@ end post '/bootstrapping' do content_type "text/uri-list" - task_uri = OpenTox::Task.as_task( "Perform bootstrapping validation", url_for("/bootstrapping", :full), params ) do |task| + task = OpenTox::Task.create( "Perform bootstrapping validation", url_for("/bootstrapping", :full) ) do |task| #, params LOGGER.info "performing bootstrapping validation "+params.inspect halt 400, "dataset_uri missing" unless params[:dataset_uri] halt 400, "algorithm_uri missing" unless params[:algorithm_uri] @@ -315,12 +340,13 @@ post '/bootstrapping' do v.validate_algorithm( params[:algorithm_params], OpenTox::SubTask.create(task,33,100)) v.validation_uri end - halt 202,task_uri+"\n" + halt 202,task.uri+"\n" end get '/bootstrapping' do LOGGER.info "list all bootstrapping-validations, params: "+params.inspect - uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "bootstrapping" } ).collect{ |v| v.validation_uri }.join("\n")+"\n" + #uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "bootstrapping" } ).collect{ |v| v.validation_uri }.join("\n")+"\n" + uri_list = Validation::Validation.all( :validation_type => "bootstrapping" ).collect{ |v| v.validation_uri }.join("\n")+"\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "All validations: "+$sinatra.url_for("/",:full)+"\n"+ @@ -343,8 +369,7 @@ end post '/training_test_split' do - content_type "text/uri-list" - task_uri = OpenTox::Task.as_task( "Perform training test split validation", url_for("/training_test_split", :full), params ) do |task| + task = OpenTox::Task.create( "Perform training test split validation", url_for("/training_test_split", :full) ) do |task| #, params LOGGER.info "creating training test split "+params.inspect halt 400, "dataset_uri missing" unless params[:dataset_uri] @@ -362,12 +387,16 @@ post '/training_test_split' do v.validate_algorithm( params[:algorithm_params], OpenTox::SubTask.create(task,33,100)) v.validation_uri end - halt 202,task_uri+"\n" + content_type 'text/uri-list' + halt 503,task.uri+"\n" if task.status == "Cancelled" + halt 202,task.uri + end get '/training_test_split' do LOGGER.info "list all training-test-split-validations, params: "+params.inspect - uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "training_test_split" } ).collect{ |v| v.validation_uri }.join("\n")+"\n" + #uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "training_test_split" } ).collect{ |v| v.validation_uri }.join("\n")+"\n" + uri_list = Validation::Validation.all( :validation_type => "training_test_split" ).collect{ |v| v.validation_uri }.join("\n")+"\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "All validations: "+$sinatra.url_for("/",:full)+"\n"+ @@ -393,10 +422,12 @@ post '/cleanup/?' do LOGGER.info "validation cleanup, starting..." content_type "text/uri-list" deleted = [] - Validation::Validation.find( :all, :conditions => { :prediction_dataset_uri => nil } ).each do |val| - LOGGER.debug "delete val with id:"+val.id.to_s+" prediction_dataset_uri is nil" + #Validation::Validation.find( :all, :conditions => { :prediction_dataset_uri => nil } ).each do |val| + Validation::Validation.all( :finished => false ).each do |val| + LOGGER.debug "delete val with id:"+val.id.to_s+", finished is false" deleted << val.validation_uri - Validation::Validation.delete(val.id) + #Validation::Validation.delete(val.id) + val.delete end LOGGER.info "validation cleanup, deleted "+deleted.size.to_s+" validations" deleted.join("\n")+"\n" @@ -413,7 +444,7 @@ end post '/validate_datasets' do content_type "text/uri-list" - task_uri = OpenTox::Task.as_task( "Perform dataset validation", url_for("/validate_datasets", :full), params ) do |task| + task = OpenTox::Task.create( "Perform dataset validation", url_for("/validate_datasets", :full) ) do |task| #, params LOGGER.info "validating values "+params.inspect halt 400, "test_dataset_uri missing" unless params[:test_dataset_uri] halt 400, "prediction_datset_uri missing" unless params[:prediction_dataset_uri] @@ -429,20 +460,21 @@ post '/validate_datasets' do params[:classification] or params[:regression] predicted_feature = params.delete("predicted_feature") - clazz = params.delete("classification")!=nil - regr = params.delete("regression")!=nil + feature_type = "classification" if params.delete("classification")!=nil + feature_type = "regression" if params.delete("regression")!=nil v = Validation::Validation.new params - v.compute_validation_stats((clazz and !regr),predicted_feature,nil,false,task) + v.compute_validation_stats(feature_type,predicted_feature,nil,nil,false,task) end v.validation_uri end - halt 202,task_uri+"\n" + halt 202,task.uri+"\n" end get '/:id/predictions' do LOGGER.info "get validation predictions "+params.inspect begin - validation = Validation::Validation.find(params[:id]) + #validation = Validation::Validation.find(params[:id]) + validation = Validation::Validation.get(params[:id]) rescue ActiveRecord::RecordNotFound => ex halt 404, "Validation '#{params[:id]}' not found." end @@ -482,7 +514,8 @@ end get '/:id' do LOGGER.info "get validation with id "+params[:id].to_s+" '"+request.env['HTTP_ACCEPT'].to_s+"'" begin - validation = Validation::Validation.find(params[:id]) + #validation = Validation::Validation.find(params[:id]) + validation = Validation::Validation.get(params[:id]) rescue ActiveRecord::RecordNotFound => ex halt 404, "Validation '#{params[:id]}' not found." end @@ -511,11 +544,13 @@ end delete '/:id' do LOGGER.info "delete validation with id "+params[:id].to_s - begin - validation = Validation::Validation.find(params[:id]) - rescue ActiveRecord::RecordNotFound => ex - halt 404, "Validation '#{params[:id]}' not found." - end +# begin + #validation = Validation::Validation.find(params[:id]) +# rescue ActiveRecord::RecordNotFound => ex +# halt 404, "Validation '#{params[:id]}' not found." +# end + validation = Validation::Validation.get(params[:id]) + halt 404, "Validation '#{params[:id]}' not found." unless validation content_type "text/plain" validation.delete end \ No newline at end of file diff --git a/validation/validation_format.rb b/validation/validation_format.rb old mode 100644 new mode 100755 index 0db8579..7c6396a --- a/validation/validation_format.rb +++ b/validation/validation_format.rb @@ -24,13 +24,14 @@ module Validation h[:crossvalidation_info] = cv end if classification_statistics + raise "classification_statistics is no has: "+classification_statistics.class.to_s unless classification_statistics.is_a?(Hash) clazz = {} Lib::VAL_CLASS_PROPS_SINGLE.each{ |p| clazz[p] = classification_statistics[p] } # transpose results per class class_values = {} Lib::VAL_CLASS_PROPS_PER_CLASS.each do |p| - $sinatra.halt 500, "missing classification statitstics: "+p.to_s+" "+classification_statistics.inspect unless classification_statistics[p] + $sinatra.halt 500, "missing classification statitstics: "+p.to_s+" "+classification_statistics.inspect if classification_statistics[p]==nil classification_statistics[p].each do |class_value, property_value| class_values[class_value] = {:class_value => class_value} unless class_values.has_key?(class_value) map = class_values[class_value] @@ -82,7 +83,8 @@ module Validation h[p] = self.send(p) end v = [] - Validation.find( :all, :conditions => { :crossvalidation_id => self.id } ).each do |val| + #Validation.find( :all, :conditions => { :crossvalidation_id => self.id } ).each do |val| + Validation.all( :crossvalidation_id => self.id ).each do |val| v.push( val.validation_uri.to_s ) end h[:validation_uris] = v diff --git a/validation/validation_service.rb b/validation/validation_service.rb old mode 100644 new mode 100755 index 7204699..01fba7b --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -1,7 +1,5 @@ -require "rdf/redland" - require "lib/validation_db.rb" require "lib/ot_predictions.rb" @@ -35,13 +33,13 @@ module Validation class Validation < Lib::Validation # constructs a validation object, Rsets id und uri - def initialize( params={} ) - $sinatra.halt 500,"do not set id manually" if params[:id] - params[:finished] = false - super params - self.save! - raise "internal error, validation-id not set "+to_yaml if self.id==nil - end + #def initialize( params={} ) + #$sinatra.halt 500,"do not set id manually" if params[:id] + #params[:finished] = false + #super params + #self.save! + #raise "internal error, validation-id not set "+to_yaml if self.id==nil + #end # deletes a validation # PENDING: model and referenced datasets are deleted as well, keep it that way? @@ -55,7 +53,8 @@ module Validation #dataset.delete if dataset #end - Validation::Validation.delete(self.id) + #Validation::Validation.delete(self.id) + self.destroy "Successfully deleted validation "+self.id.to_s+"." end @@ -76,14 +75,22 @@ module Validation end LOGGER.debug "building model '"+algorithm_uri.to_s+"' "+params.inspect - model = OpenTox::Model::PredictionModel.build(algorithm_uri, params, - OpenTox::SubTask.create(task, 0, 33) ) - $sinatra.halt 500,"model building failed" unless model - self.attributes = { :model_uri => model.uri } - self.save! + algorithm = OpenTox::Algorithm::Generic.new(algorithm_uri) + self.model_uri = algorithm.run(params) + task.progress(33) + + #model = OpenTox::Model::PredictionModel.build(algorithm_uri, params, + # OpenTox::SubTask.create(task, 0, 33) ) - $sinatra.halt 500,"error after building model: model.dependent_variable != validation.prediciton_feature ("+ - model.dependentVariables.to_s+" != "+self.prediction_feature+")" if self.prediction_feature!=model.dependentVariables + $sinatra.halt 500,"model building failed" unless model_uri + #self.attributes = { :model_uri => model_uri } + #self.save! + +# self.save if self.new? +# self.update :model_uri => model_uri + + #$sinatra.halt 500,"error after building model: model.dependent_variable != validation.prediciton_feature ("+ + # model.dependentVariables.to_s+" != "+self.prediction_feature+")" if self.prediction_feature!=model.dependentVariables validate_model OpenTox::SubTask.create(task, 33, 100) end @@ -95,70 +102,103 @@ module Validation $sinatra.halt 500,"validation_type missing" unless self.validation_type LOGGER.debug "validating model '"+self.model_uri+"'" - model = OpenTox::Model::PredictionModel.find(self.model_uri) + #model = OpenTox::Model::PredictionModel.find(self.model_uri) + #$sinatra.halt 400, "model not found: "+self.model_uri.to_s unless model + model = OpenTox::Model::Generic.find(self.model_uri) $sinatra.halt 400, "model not found: "+self.model_uri.to_s unless model unless self.algorithm_uri - self.attributes = { :algorithm_uri => model.algorithm } - self.save! +# self.attributes = { :algorithm_uri => model.algorithm } +# self.save! + #self.update :algorithm_uri => model.algorithm + self.algorithm_uri = model.metadata[OT.algorithm] end + dependentVariables = model.metadata[OT.dependentVariables] if self.prediction_feature - $sinatra.halt 400, "error validating model: model.dependent_variable != validation.prediciton_feature ("+ - model.dependentVariables+" != "+self.prediction_feature+")" if self.prediction_feature!=model.dependentVariables + $sinatra.halt 400, "error validating model: model.dependent_variable != validation.prediction_feature ("+ + dependentVariables.to_s+" != "+self.prediction_feature+"), model-metadata is "+model.metadata.inspect if self.prediction_feature!=dependentVariables else - $sinatra.halt 400, "model has no dependentVariables specified, please give prediction feature for model validation" unless model.dependentVariables - self.attributes = { :prediction_feature => model.dependentVariables } - self.save! + $sinatra.halt 400, "model has no dependentVariables specified, please give prediction feature for model validation" unless dependentVariables + #self.attributes = { :prediction_feature => model.dependentVariables } + #self.save! + #self.update :prediction_feature => model.dependentVariables + self.prediction_feature = model.metadata[OT.dependentVariables] end prediction_dataset_uri = "" benchmark = Benchmark.measure do - prediction_dataset_uri = model.predict_dataset(self.test_dataset_uri, OpenTox::SubTask.create(task, 0, 50)) + #prediction_dataset_uri = model.predict_dataset(self.test_dataset_uri, OpenTox::SubTask.create(task, 0, 50)) + prediction_dataset_uri = model.run(:dataset_uri => self.test_dataset_uri) + task.progress(50) end - self.attributes = { :prediction_dataset_uri => prediction_dataset_uri, - :real_runtime => benchmark.real } - self.save! - +# self.attributes = { :prediction_dataset_uri => prediction_dataset_uri, +# :real_runtime => benchmark.real } +# self.save! +# self.update :prediction_dataset_uri => prediction_dataset_uri, +# :real_runtime => benchmark.real + self.prediction_dataset_uri = prediction_dataset_uri + self.real_runtime = benchmark.real + compute_validation_stats_with_model( model, false, OpenTox::SubTask.create(task, 50, 100) ) end def compute_validation_stats_with_model( model=nil, dry_run=false, task=nil ) - model = OpenTox::Model::PredictionModel.find(self.model_uri) if model==nil and self.model_uri + #model = OpenTox::Model::PredictionModel.find(self.model_uri) if model==nil and self.model_uri + #$sinatra.halt 400, "model not found: "+self.model_uri.to_s unless model + model = OpenTox::Model::Generic.find(self.model_uri) if model==nil and self.model_uri $sinatra.halt 400, "model not found: "+self.model_uri.to_s unless model - prediction_feature = self.prediction_feature ? nil : model.dependentVariables - algorithm_uri = self.algorithm_uri ? nil : model.algorithm - compute_validation_stats( model.classification?, model.predictedVariables, + + dependentVariables = model.metadata[OT.dependentVariables] + prediction_feature = self.prediction_feature ? nil : dependentVariables + algorithm_uri = self.algorithm_uri ? nil : model.metadata[OT.algorithm] + predictedVariables = model.metadata[OT.predictedVariables] + compute_validation_stats( model.feature_type, predictedVariables, prediction_feature, algorithm_uri, dry_run, task ) end - def compute_validation_stats( classification, predicted_feature, prediction_feature=nil, + def compute_validation_stats( feature_type, predicted_feature, prediction_feature=nil, algorithm_uri=nil, dry_run=false, task=nil ) - self.attributes = { :prediction_feature => prediction_feature } if self.prediction_feature==nil && prediction_feature - self.attributes = { :algorithm_uri => algorithm_uri } if self.algorithm_uri==nil && algorithm_uri - self.save! +# self.attributes = { :prediction_feature => prediction_feature } if self.prediction_feature==nil && prediction_feature +# self.attributes = { :algorithm_uri => algorithm_uri } if self.algorithm_uri==nil && algorithm_uri +# self.save! +# self.update :prediction_feature => prediction_feature if self.prediction_feature==nil && prediction_feature +# self.update :algorithm_uri => algorithm_uri if self.algorithm_uri==nil && algorithm_uri + self.prediction_feature = prediction_feature if self.prediction_feature==nil && prediction_feature + self.algorithm_uri = algorithm_uri if self.algorithm_uri==nil && algorithm_uri LOGGER.debug "computing prediction stats" - prediction = Lib::OTPredictions.new( classification, + prediction = Lib::OTPredictions.new( feature_type, self.test_dataset_uri, self.test_target_dataset_uri, self.prediction_feature, self.prediction_dataset_uri, predicted_feature, OpenTox::SubTask.create(task, 0, 80) ) #reading datasets and computing the main stats is 80% the work - + unless dry_run - if prediction.classification? - self.attributes = { :classification_statistics => prediction.compute_stats } - else - self.attributes = { :regression_statistics => prediction.compute_stats } + case feature_type + when "classification" + #self.attributes = { :classification_statistics => prediction.compute_stats } + #self.update :classification_statistics => prediction.compute_stats + self.classification_statistics = prediction.compute_stats + when "regression" + #self.attributes = { :regression_statistics => prediction.compute_stats } + self.regression_statistics = prediction.compute_stats end - self.attributes = { :num_instances => prediction.num_instances, +# self.attributes = { :num_instances => prediction.num_instances, +# :num_without_class => prediction.num_without_class, +# :percent_without_class => prediction.percent_without_class, +# :num_unpredicted => prediction.num_unpredicted, +# :percent_unpredicted => prediction.percent_unpredicted, +# :finished => true} +# self.save! + self.attributes= {:num_instances => prediction.num_instances, :num_without_class => prediction.num_without_class, :percent_without_class => prediction.percent_without_class, :num_unpredicted => prediction.num_unpredicted, :percent_unpredicted => prediction.percent_unpredicted, :finished => true} - self.save! + self.save end task.progress(100) if task @@ -169,17 +209,17 @@ module Validation class Crossvalidation < Lib::Crossvalidation # constructs a crossvalidation, id and uri are set - def initialize( params={} ) - - $sinatra.halt 500,"do not set id manually" if params[:id] - params[:num_folds] = 10 if params[:num_folds]==nil - params[:random_seed] = 1 if params[:random_seed]==nil - params[:stratified] = false if params[:stratified]==nil - params[:finished] = false - super params - self.save! - raise "internal error, crossvalidation-id not set" if self.id==nil - end + #def initialize( params={} ) + # + # $sinatra.halt 500,"do not set id manually" if params[:id] + # params[:num_folds] = 10 if params[:num_folds]==nil + # params[:random_seed] = 1 if params[:random_seed]==nil + # params[:stratified] = false if params[:stratified]==nil + # params[:finished] = false + # super params + # self.save! + # raise "internal error, crossvalidation-id not set" if self.id==nil + #end def perform_cv ( prediction_feature, algorithm_params=nil, task=nil ) @@ -189,7 +229,7 @@ module Validation # deletes a crossvalidation, all validations are deleted as well def delete - Validation.all(:crossvalidation_id => self.id).each{ |v| v.delete } + Validation.all(:crossvalidation_id => self.id).each{ |v| v.destroy } destroy "Successfully deleted crossvalidation "+self.id.to_s+"." end @@ -216,8 +256,11 @@ module Validation i += 1 end - self.attributes = { :finished => true } - self.save! +# self.attributes = { :finished => true } +# self.save! + #self.save if self.new? + self.finished = true + self.save end private @@ -255,7 +298,7 @@ module Validation # stores uris in validation objects def create_new_cv_datasets( prediction_feature, task = nil ) - $sinatra.halt(500,"random seed not set") unless self.random_seed + $sinatra.halt(500,"random seed not set "+self.inspect) unless self.random_seed LOGGER.debug "creating datasets for crossvalidation" orig_dataset = OpenTox::Dataset.find(self.dataset_uri) $sinatra.halt 400, "Dataset not found: "+self.dataset_uri.to_s unless orig_dataset @@ -302,7 +345,7 @@ module Validation end LOGGER.debug "cv: num instances for each fold: "+split_compounds.collect{|c| c.size}.join(", ") - test_features = orig_dataset.features.dclone - [prediction_feature] + test_features = orig_dataset.features.keys.dclone - [prediction_feature] @tmp_validations = [] @@ -332,11 +375,17 @@ module Validation $sinatra.halt 500,"internal error, num train compounds not correct" unless shuffled_compounds.size - test_compounds.size == train_compounds.size LOGGER.debug "training set: "+datasetname+"_train, compounds: "+train_compounds.size.to_s - train_dataset_uri = orig_dataset.create_new_dataset( train_compounds, orig_dataset.features, datasetname + '_train', source ) + #train_dataset_uri = orig_dataset.create_new_dataset( train_compounds, orig_dataset.features, datasetname + '_train', source ) + train_dataset_uri = orig_dataset.split( train_compounds, orig_dataset.features.keys, + { DC.title => datasetname + '_train', DC.creator => source } ).uri LOGGER.debug "test set: "+datasetname+"_test, compounds: "+test_compounds.size.to_s - test_dataset_uri = orig_dataset.create_new_dataset( test_compounds, test_features, datasetname + '_test', source ) - + #test_dataset_uri = orig_dataset.create_new_dataset( test_compounds, test_features, datasetname + '_test', source ) + test_dataset_uri = orig_dataset.split( test_compounds, test_features, + { DC.title => datasetname + '_test', DC.creator => source } ).uri + + #make sure self.id is set + self.save if self.new? tmp_validation = { :validation_type => "crossvalidation", :training_dataset_uri => train_dataset_uri, :test_dataset_uri => test_dataset_uri, @@ -362,6 +411,7 @@ module Validation random_seed=1 unless random_seed orig_dataset = OpenTox::Dataset.find orig_dataset_uri + orig_dataset.load_all $sinatra.halt 400, "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset if prediction_feature $sinatra.halt 400, "Prediction feature '"+prediction_feature.to_s+ @@ -374,6 +424,11 @@ module Validation compounds = orig_dataset.compounds $sinatra.halt 400, "Cannot split datset, num compounds in dataset < 2 ("+compounds.size.to_s+")" if compounds.size<2 + compounds.each do |c| + $sinatra.halt 400, "Bootstrapping not yet implemented for duplicate compounds" if + orig_dataset.data_entries[c][prediction_feature].size > 1 + end + srand random_seed.to_i while true training_compounds = [] @@ -397,23 +452,35 @@ module Validation task.progress(33) if task result = {} - result[:training_dataset_uri] = orig_dataset.create_new_dataset( training_compounds, - orig_dataset.features, - "Bootstrapping training dataset of "+orig_dataset.title.to_s, - $sinatra.url_for('/bootstrapping',:full) ) +# result[:training_dataset_uri] = orig_dataset.create_new_dataset( training_compounds, +# orig_dataset.features, +# "Bootstrapping training dataset of "+orig_dataset.title.to_s, +# $sinatra.url_for('/bootstrapping',:full) ) + result[:training_dataset_uri] = orig_dataset.split( training_compounds, + orig_dataset.features.keys, + { DC.title => "Bootstrapping training dataset of "+orig_dataset.title.to_s, + DC.creator => $sinatra.url_for('/bootstrapping',:full) }).uri task.progress(66) if task - result[:test_dataset_uri] = orig_dataset.create_new_dataset( test_compounds, - orig_dataset.features.dclone - [prediction_feature], - "Bootstrapping test dataset of "+orig_dataset.title.to_s, - $sinatra.url_for('/bootstrapping',:full) ) +# result[:test_dataset_uri] = orig_dataset.create_new_dataset( test_compounds, +# orig_dataset.features.dclone - [prediction_feature], +# "Bootstrapping test dataset of "+orig_dataset.title.to_s, +# $sinatra.url_for('/bootstrapping',:full) ) + result[:test_dataset_uri] = orig_dataset.split( test_compounds, + orig_dataset.features.keys.dclone - [prediction_feature], + { DC.title => "Bootstrapping test dataset of "+orig_dataset.title.to_s, + DC.creator => $sinatra.url_for('/bootstrapping',:full) }).uri task.progress(100) if task if ENV['RACK_ENV'] =~ /test|debug/ training_dataset = OpenTox::Dataset.find result[:training_dataset_uri] $sinatra.halt 400, "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless training_dataset - training_compounds_verify = training_dataset.compounds - $sinatra.halt 500, "training compounds error" unless training_compounds_verify==training_compounds + training_dataset.load_all + value_count = 0 + training_dataset.compounds.each do |c| + value_count += training_dataset.data_entries[c][prediction_feature].size + end + $sinatra.halt 500, "training compounds error" unless value_count==training_compounds.size $sinatra.halt 400, "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless OpenTox::Dataset.find result[:test_dataset_uri] end LOGGER.debug "bootstrapping done, training dataset: '"+result[:training_dataset_uri].to_s+"', test dataset: '"+result[:test_dataset_uri].to_s+"'" @@ -429,13 +496,14 @@ module Validation random_seed=1 unless random_seed orig_dataset = OpenTox::Dataset.find orig_dataset_uri + orig_dataset.load_all $sinatra.halt 400, "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset $sinatra.halt 400, "Split ratio invalid: "+split_ratio.to_s unless split_ratio and split_ratio=split_ratio.to_f $sinatra.halt 400, "Split ratio not >0 and <1 :"+split_ratio.to_s unless split_ratio>0 && split_ratio<1 if prediction_feature $sinatra.halt 400, "Prediction feature '"+prediction_feature.to_s+ "' not found in dataset, features are: \n"+ - orig_dataset.features.inspect unless orig_dataset.features.include?(prediction_feature) + orig_dataset.features.keys.inspect unless orig_dataset.features.include?(prediction_feature) else LOGGER.warn "no prediciton feature given, all features included in test dataset" end @@ -453,21 +521,46 @@ module Validation task.progress(33) if task result = {} - result[:training_dataset_uri] = orig_dataset.create_new_dataset( compounds[0..split], - orig_dataset.features, - "Training dataset split of "+orig_dataset.title.to_s, - $sinatra.url_for('/training_test_split',:full) ) +# result[:training_dataset_uri] = orig_dataset.create_new_dataset( compounds[0..split], +# orig_dataset.features, +# "Training dataset split of "+orig_dataset.title.to_s, +# $sinatra.url_for('/training_test_split',:full) ) + +# orig_dataset.data_entries.each do |k,v| +# puts k.inspect+" =>"+v.inspect +# puts v.values[0].to_s+" "+v.values[0].class.to_s +# end + + result[:training_dataset_uri] = orig_dataset.split( compounds[0..split], + orig_dataset.features.keys, + { DC.title => "Training dataset split of "+orig_dataset.title.to_s, + DC.creator => $sinatra.url_for('/training_test_split',:full) } ).uri task.progress(66) if task - - result[:test_dataset_uri] = orig_dataset.create_new_dataset( compounds[(split+1)..-1], - orig_dataset.features.dclone - [prediction_feature], - "Test dataset split of "+orig_dataset.title.to_s, - $sinatra.url_for('/training_test_split',:full) ) + +# d = OpenTox::Dataset.find(result[:training_dataset_uri]) +# d.data_entries.values.each do |v| +# puts v.inspect +# puts v.values[0].to_s+" "+v.values[0].class.to_s +# end +# raise "stop here" + +# result[:test_dataset_uri] = orig_dataset.create_new_dataset( compounds[(split+1)..-1], +# orig_dataset.features.dclone - [prediction_feature], +# "Test dataset split of "+orig_dataset.title.to_s, +# $sinatra.url_for('/training_test_split',:full) ) + result[:test_dataset_uri] = orig_dataset.split( compounds[(split+1)..-1], + orig_dataset.features.keys.dclone - [prediction_feature], + { DC.title => "Test dataset split of "+orig_dataset.title.to_s, + DC.creator => $sinatra.url_for('/training_test_split',:full) } ).uri task.progress(100) if task if ENV['RACK_ENV'] =~ /test|debug/ $sinatra.halt 400, "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless OpenTox::Dataset.find result[:training_dataset_uri] - $sinatra.halt 400, "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless OpenTox::Dataset.find result[:test_dataset_uri] + test_data = OpenTox::Dataset.find result[:test_dataset_uri] + $sinatra.halt 400, "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless test_data + test_data.load_compounds + $sinatra.halt 400, "Test dataset num coumpounds != "+(compounds.size-split-1).to_s+", instead: "+ + test_data.compounds.size.to_s+"\n"+test_data.to_yaml unless test_data.compounds.size==(compounds.size-1-split) end LOGGER.debug "split done, training dataset: '"+result[:training_dataset_uri].to_s+"', test dataset: '"+result[:test_dataset_uri].to_s+"'" diff --git a/validation/validation_test.rb b/validation/validation_test.rb old mode 100644 new mode 100755 index b1fbf25..d9c2fbb --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -7,7 +7,7 @@ require 'rack/test' require 'lib/test_util.rb' require 'test/test_examples.rb' -LOGGER = MyLogger.new(STDOUT) +LOGGER = OTLogger.new(STDOUT) LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S " LOGGER.formatter = Logger::Formatter.new @@ -15,7 +15,7 @@ LOGGER.formatter = Logger::Formatter.new module Sinatra module UrlForHelper BASE = "http://localhost/validation" - def url_for url_fragment, mode=:path_only + def url_for url_fragment, mode=:path_only case mode when :path_only raise "not impl" @@ -35,8 +35,9 @@ class ValidationTest < Test::Unit::TestCase $test_case = self #get "/1",nil,'HTTP_ACCEPT' => "text/html" -# get "/",nil,'HTTP_ACCEPT' => "text/html" + #get "/2",nil,'HTTP_ACCEPT' => "application/rdf+xml" #puts last_response.body + #exit # d = OpenTox::Dataset.find("http://ot-dev.in-silico.ch/dataset/307") # puts d.compounds.inspect @@ -59,12 +60,13 @@ class ValidationTest < Test::Unit::TestCase #post "/crossvalidation/cleanup" #puts last_response.body - #get "/crossvalidation/4/statistics" + #get "/crossvalidation/19/predictions",nil,'HTTP_ACCEPT' => "application/x-yaml" #/statistics" # post "",:model_uri=>"http://localhost/model/1",:test_dataset_uri=>"http://localhost/dataset/3", # :test_target_dataset_uri=>"http://localhost/dataset/1" # get "/crossvalidation/2",nil,'HTTP_ACCEPT' => "application/rdf+xml" - #puts last_response.body + #puts last_response.body + #exit #get "/crossvalidation?model_uri=lazar" # post "/test_validation",:select=>"6d" #,:report=>"yes,please" @@ -79,8 +81,32 @@ class ValidationTest < Test::Unit::TestCase # :regression=>"true"} # #:classification=>"true"} # puts last_response.body - - #run_test("1b","http://localhost/validation/28")#,"http://localhost/validation/394"); + +# post "/validate_datasets",{ +# :test_dataset_uri=>"http://localhost/dataset/89", +# :prediction_dataset_uri=>"http://localhost/dataset/91", +# :test_target_dataset_uri=>"http://localhost/dataset/87", +# :prediction_feature=>"http://localhost/dataset/1/feature/hamster_carcinogenicity", +# :predicted_feature=>"", +## :regression=>"true"} +# :classification=>"true"} +# puts last_response.body + + # m = OpenTox::Model::Generic.find("http://localhost/model/1323333") + # puts m.to_yaml + +# post "/validate_datasets",{ +# :test_dataset_uri=>"http://localhost/dataset/150", +# :prediction_dataset_uri=>"http://localhost/dataset/152", +# :test_target_dataset_uri=>"http://localhost/dataset/148", +# :prediction_feature=>"http://localhost/dataset/148/feature/LC50_mmol", +# :model_uri=>"http://localhost/model/13"} +# #:regression=>"true"} +## :classification=>"true"} +# puts last_response.body + + #run_test("13a","http://localhost/validation/39",nil,false) #,"http://localhost/validation/28")#,"http://localhost/validation/394"); + run_test("1b",nil,nil,false) #run_test("7b","http://localhost/validation/21") @@ -96,7 +122,7 @@ class ValidationTest < Test::Unit::TestCase #puts Nightly.build_nightly("1") - prepare_examples + #prepare_examples #do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE #do_test_examples_ortona end @@ -105,7 +131,7 @@ class ValidationTest < Test::Unit::TestCase Sinatra::Application end - def run_test(select=nil, validation_uri=nil, report_uri=nil) + def run_test(select=nil, validation_uri=nil, report_uri=nil, delete=false) puts ValidationExamples.list unless select validationExamples = ValidationExamples.select(select) validationExamples.each do |vv| @@ -122,8 +148,9 @@ class ValidationTest < Test::Unit::TestCase unless ex.report_uri ex.report end - ex.verify_yaml - ex.compare_yaml_vs_rdf + #ex.verify_yaml + #ex.compare_yaml_vs_rdf + ex.delete if delete end end end -- cgit v1.2.3