diff options
author | mguetlein <martin.guetlein@gmail.com> | 2011-05-17 10:46:45 +0200 |
---|---|---|
committer | mguetlein <martin.guetlein@gmail.com> | 2011-05-17 10:46:45 +0200 |
commit | 9ce03c0f50bb9129b584327d56fa4c9277849227 (patch) | |
tree | 8c0213ec8e3e5ac2ca918ab03a78c6fa99f2fcdc | |
parent | eb5f8b5da9b247d62abc8a7b9eb2e44fe46a1c79 (diff) |
crossvalidation statistics fix: compute cv-statistics with cv-predictions instead of averaging cv-validation-statistics
-rw-r--r-- | lib/dataset_cache.rb | 23 | ||||
-rwxr-xr-x | lib/ot_predictions.rb | 254 | ||||
-rwxr-xr-x | reach_reports/reach_service.rb | 47 | ||||
-rw-r--r-- | report/plot_factory.rb | 2 | ||||
-rwxr-xr-x | report/report_content.rb | 14 | ||||
-rwxr-xr-x | report/validation_access.rb | 32 | ||||
-rwxr-xr-x | report/validation_data.rb | 39 | ||||
-rwxr-xr-x | validation/validation_application.rb | 35 | ||||
-rwxr-xr-x | validation/validation_format.rb | 2 | ||||
-rwxr-xr-x | validation/validation_service.rb | 84 |
10 files changed, 322 insertions, 210 deletions
diff --git a/lib/dataset_cache.rb b/lib/dataset_cache.rb new file mode 100644 index 0000000..1af1d51 --- /dev/null +++ b/lib/dataset_cache.rb @@ -0,0 +1,23 @@ + +module Lib + + module DatasetCache + + @@cache={} + + # same as OpenTox::Dataset.find with caching function + # rational: datasets are reused in crossvalidation very often, cache to save computational effort + # PENDING: may cause memory issues, test with huge datasets + def self.find(dataset_uri, subjectid=nil) + return nil if (dataset_uri==nil) + d = @@cache[dataset_uri.to_s+"_"+subjectid.to_s] + if d==nil + d = OpenTox::Dataset.find(dataset_uri, subjectid) + @@cache[dataset_uri.to_s+"_"+subjectid.to_s] = d + end + d + end + + end + +end
\ No newline at end of file diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index 22f9b20..644168f 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -15,127 +15,161 @@ module Lib return @compounds[instance_index] end - def initialize(feature_type, test_dataset_uri, test_target_dataset_uri, - prediction_feature, prediction_dataset_uri, predicted_variable, subjectid=nil, task=nil) + def initialize( feature_type, test_dataset_uris, test_target_dataset_uris, + prediction_feature, prediction_dataset_uris, predicted_variables, subjectid=nil, task=nil) - LOGGER.debug("loading prediciton via test-dataset:'"+test_dataset_uri.to_s+ - "', test-target-datset:'"+test_target_dataset_uri.to_s+ - "', prediction-dataset:'"+prediction_dataset_uri.to_s+ - "', prediction_feature: '"+prediction_feature.to_s+"' "+ - "', predicted_variable: '"+predicted_variable.to_s+"'") - - predicted_variable=prediction_feature if predicted_variable==nil - - test_dataset = OpenTox::Dataset.find test_dataset_uri,subjectid - raise "test dataset not found: '"+test_dataset_uri.to_s+"'" unless test_dataset + test_dataset_uris = [test_dataset_uris] unless test_dataset_uris.is_a?(Array) + test_target_dataset_uris = [test_target_dataset_uris] unless test_target_dataset_uris.is_a?(Array) + prediction_dataset_uris = [prediction_dataset_uris] unless prediction_dataset_uris.is_a?(Array) + predicted_variables = [predicted_variables] unless predicted_variables.is_a?(Array) + LOGGER.debug "loading prediciton -- test-dataset: "+test_dataset_uris.inspect + LOGGER.debug "loading prediciton -- test-target-datset: "+test_target_dataset_uris.inspect + LOGGER.debug "loading prediciton -- prediction-dataset: "+prediction_dataset_uris.inspect + LOGGER.debug "loading prediciton -- predicted_variable: "+predicted_variables.inspect + LOGGER.debug "loading prediciton -- prediction_feature: "+prediction_feature.to_s raise "prediction_feature missing" unless prediction_feature - if test_target_dataset_uri == nil || test_target_dataset_uri.strip.size==0 || test_target_dataset_uri==test_dataset_uri - test_target_dataset_uri = test_dataset_uri - test_target_dataset = test_dataset - raise "prediction_feature not found in test_dataset, specify a test_target_dataset\n"+ - "prediction_feature: '"+prediction_feature.to_s+"'\n"+ - "test_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ - "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil - else - test_target_dataset = OpenTox::Dataset.find test_target_dataset_uri,subjectid - raise "test target datset not found: '"+test_target_dataset_uri.to_s+"'" unless test_target_dataset - if CHECK_VALUES - test_dataset.compounds.each do |c| - raise "test compound not found on test class dataset "+c.to_s unless test_target_dataset.compounds.include?(c) - end - end - raise "prediction_feature not found in test_target_dataset\n"+ - "prediction_feature: '"+prediction_feature.to_s+"'\n"+ - "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ - "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil - end - - @compounds = test_dataset.compounds - LOGGER.debug "test dataset size: "+@compounds.size.to_s - raise "test dataset is empty "+test_dataset_uri.to_s unless @compounds.size>0 - - if feature_type=="classification" - accept_values = test_target_dataset.features[prediction_feature][OT.acceptValue] - raise "'"+OT.acceptValue.to_s+"' missing/invalid for feature '"+prediction_feature.to_s+"' in dataset '"+ - test_target_dataset_uri.to_s+"', acceptValues are: '"+accept_values.inspect+"'" if accept_values==nil or accept_values.length<2 - else - accept_values=nil - end + @compounds = [] + all_predicted_values = [] + all_actual_values = [] + all_confidence_values = [] + accept_values = nil - actual_values = [] - @compounds.each do |c| - case feature_type - when "classification" - actual_values << classification_value(test_target_dataset, c, prediction_feature, accept_values) - when "regression" - actual_values << regression_value(test_target_dataset, c, prediction_feature) - end + if task + task_step = 100 / (test_dataset_uris.size*2 + 1) + task_status = 0 end - task.progress(40) if task # loaded actual values + + test_dataset_uris.size.times do |i| + + test_dataset_uri = test_dataset_uris[i] + test_target_dataset_uri = test_target_dataset_uris[i] + prediction_dataset_uri = prediction_dataset_uris[i] + predicted_variable = predicted_variables[i] + + predicted_variable=prediction_feature if predicted_variable==nil - prediction_dataset = OpenTox::Dataset.find prediction_dataset_uri,subjectid - raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset + test_dataset = Lib::DatasetCache.find test_dataset_uri,subjectid + raise "test dataset not found: '"+test_dataset_uri.to_s+"'" unless test_dataset - # TODO: remove LAZAR_PREDICTION_DATASET_HACK - no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil - if no_prediction_feature - one_entry_per_compound = true - @compounds.each do |c| - if prediction_dataset.data_entries[c] and prediction_dataset.data_entries[c].size != 1 - one_entry_per_compound = false - break - end - end - msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+ - prediction_dataset.features.keys.inspect - if one_entry_per_compound - LOGGER.warn msg + if test_target_dataset_uri == nil || test_target_dataset_uri.strip.size==0 || test_target_dataset_uri==test_dataset_uri + test_target_dataset_uri = test_dataset_uri + test_target_dataset = test_dataset + raise "prediction_feature not found in test_dataset, specify a test_target_dataset\n"+ + "prediction_feature: '"+prediction_feature.to_s+"'\n"+ + "test_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ + "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil else - raise msg + test_target_dataset = Lib::DatasetCache.find test_target_dataset_uri,subjectid + raise "test target datset not found: '"+test_target_dataset_uri.to_s+"'" unless test_target_dataset + if CHECK_VALUES + test_dataset.compounds.each do |c| + raise "test compound not found on test class dataset "+c.to_s unless test_target_dataset.compounds.include?(c) + end + end + raise "prediction_feature not found in test_target_dataset\n"+ + "prediction_feature: '"+prediction_feature.to_s+"'\n"+ + "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ + "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil end - end - - raise "more predicted than test compounds test:"+@compounds.size.to_s+" < prediction:"+ - prediction_dataset.compounds.size.to_s if @compounds.size < prediction_dataset.compounds.size - if CHECK_VALUES - prediction_dataset.compounds.each do |c| - raise "predicted compound not found in test dataset:\n"+c+"\ntest-compounds:\n"+ - @compounds.collect{|c| c.to_s}.join("\n") if @compounds.index(c)==nil + + compounds = test_dataset.compounds + LOGGER.debug "test dataset size: "+compounds.size.to_s + raise "test dataset is empty "+test_dataset_uri.to_s unless compounds.size>0 + + if feature_type=="classification" + av = test_target_dataset.features[prediction_feature][OT.acceptValue] + raise "'"+OT.acceptValue.to_s+"' missing/invalid for feature '"+prediction_feature.to_s+"' in dataset '"+ + test_target_dataset_uri.to_s+"', acceptValues are: '"+av.inspect+"'" if av==nil or av.length<2 + if accept_values==nil + accept_values=av + else + raise "accept values (in folds) differ "+av.inspect+" != "+accept_values.inspect if av!=accept_values + end end - end - - predicted_values = [] - confidence_values = [] - @compounds.each do |c| - if prediction_dataset.compounds.index(c)==nil - predicted_values << nil - confidence_values << nil - else + + actual_values = [] + compounds.each do |c| case feature_type when "classification" - # TODO: remove LAZAR_PREDICTION_DATASET_HACK - predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, accept_values) + actual_values << classification_value(test_target_dataset, c, prediction_feature, accept_values) when "regression" - predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable) + actual_values << regression_value(test_target_dataset, c, prediction_feature) + end + end + task.progress( task_status += task_step ) if task # loaded actual values + + prediction_dataset = Lib::DatasetCache.find prediction_dataset_uri,subjectid + raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset + + # TODO: remove LAZAR_PREDICTION_DATASET_HACK + no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil + if no_prediction_feature + one_entry_per_compound = true + compounds.each do |c| + if prediction_dataset.data_entries[c] and prediction_dataset.data_entries[c].size != 1 + one_entry_per_compound = false + break + end + end + msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+ + prediction_dataset.features.keys.inspect + if one_entry_per_compound + LOGGER.warn msg + else + raise msg end - # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable) - conf = 1 - begin - feature = prediction_dataset.data_entries[c].keys[0] - feature_data = prediction_dataset.features[feature] - conf = feature_data[OT.confidence] if feature_data[OT.confidence]!=nil - rescue - LOGGER.warn "could not get confidence" + end + + raise "more predicted than test compounds test:"+compounds.size.to_s+" < prediction:"+ + prediction_dataset.compounds.size.to_s if compounds.size < prediction_dataset.compounds.size + if CHECK_VALUES + prediction_dataset.compounds.each do |c| + raise "predicted compound not found in test dataset:\n"+c+"\ntest-compounds:\n"+ + compounds.collect{|c| c.to_s}.join("\n") if compounds.index(c)==nil end - confidence_values << conf end + + predicted_values = [] + confidence_values = [] + count = 0 + compounds.each do |c| + if prediction_dataset.compounds.index(c)==nil + predicted_values << nil + confidence_values << nil + else + case feature_type + when "classification" + # TODO: remove LAZAR_PREDICTION_DATASET_HACK + predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, accept_values) + when "regression" + predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable) + end + # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable) + conf = predicted_values[count]!=nil ? 1 : 0 + begin + feature = prediction_dataset.data_entries[c].keys[0] + feature_data = prediction_dataset.features[feature] + conf = feature_data[OT.confidence] if feature_data[OT.confidence]!=nil + rescue + LOGGER.warn "could not get confidence" + end + confidence_values << conf + end + count += 1 + end + + @compounds += compounds + all_predicted_values += predicted_values + all_actual_values += actual_values + all_confidence_values += confidence_values + + task.progress( task_status += task_step ) if task # loaded predicted values and confidence end - task.progress(80) if task # loaded predicted values and confidence - super(predicted_values, actual_values, confidence_values, feature_type, accept_values) - raise "illegal num compounds "+num_info if @compounds.size != @predicted_values.size - task.progress(100) if task # done with the mathmatics + super(all_predicted_values, all_actual_values, all_confidence_values, feature_type, accept_values) + raise "illegal num compounds "+num_info if @compounds.size != @predicted_values.size + task.progress(100) if task # done with the mathmatics end private @@ -205,6 +239,7 @@ module Lib def self.to_array( predictions, add_pic=false, format=false ) res = [] + conf_column = nil predictions.each do |p| (0..p.num_instances-1).each do |i| a = [] @@ -230,13 +265,22 @@ module Lib end end if p.confidence_values_available? - a << (format ? p.confidence_value(i).to_nice_s : p.confidence_value(i)) + conf_column = a.size + a << p.confidence_value(i) #(format ? p.confidence_value(i).to_nice_s : p.confidence_value(i)) end a << p.identifier(i) res << a end end - + + if conf_column!=nil + res.sort!{ |x,y| y[4] <=> x[4] } + if format + res.each do |a| + a[4] = a[4].to_nice_s + end + end + end header = [] header << "compound" if add_pic header << "actual value" diff --git a/reach_reports/reach_service.rb b/reach_reports/reach_service.rb index 0cf4172..b6c6350 100755 --- a/reach_reports/reach_service.rb +++ b/reach_reports/reach_service.rb @@ -162,33 +162,36 @@ module ReachReports next if cvs.size==0 lmo << "crossvalidation/s on "+desc cvs.each do |cv| - lmo << "crossvalidation: "+cv.crossvalidation_uri - lmo << "dataset (see 9.3 Validation data): "+cv.dataset_uri - val_datasets << cv.dataset_uri - lmo << "settings: num-folds="+cv.num_folds.to_s+", random-seed="+cv.random_seed.to_s+", stratified:"+cv.stratified.to_s - - val = YAML.load( OpenTox::RestClientWrapper.get(File.join(cv.crossvalidation_uri,"statistics"),{:subjectid => r.subjectid}) ) - case feature_type - when "classification" - lmo << "percent_correct: "+val[OT.classificationStatistics][OT.percentCorrect].to_s - lmo << "weighted AUC: "+val[OT.classificationStatistics][OT.weightedAreaUnderRoc].to_s - when "regression" - lmo << "root_mean_squared_error: "+val[OT.regressionStatistics][OT.rootMeanSquaredError].to_s - lmo << "r_square "+val[OT.regressionStatistics][OT.rSquare].to_s - end - reports = OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"], - "report/crossvalidation?crossvalidation_uris="+cv.crossvalidation_uri),{:subjectid => r.subjectid}) - if reports and reports.chomp.size>0 - lmo << "for more info see report: "+reports.split("\n")[0] - else - lmo << "for more info see report: not yet created for '"+cv.crossvalidation_uri+"'" + begin + lmo << "crossvalidation: "+cv.crossvalidation_uri + lmo << "dataset (see 9.3 Validation data): "+cv.dataset_uri + val_datasets << cv.dataset_uri + lmo << "settings: num-folds="+cv.num_folds.to_s+", random-seed="+cv.random_seed.to_s+", stratified:"+cv.stratified.to_s + + val = YAML.load( OpenTox::RestClientWrapper.get(File.join(cv.crossvalidation_uri,"statistics"),{:subjectid => r.subjectid}) ) + case feature_type + when "classification" + lmo << "percent_correct: "+val[OT.classificationStatistics][OT.percentCorrect].to_s + lmo << "weighted AUC: "+val[OT.classificationStatistics][OT.weightedAreaUnderRoc].to_s + when "regression" + lmo << "root_mean_squared_error: "+val[OT.regressionStatistics][OT.rootMeanSquaredError].to_s + lmo << "r_square "+val[OT.regressionStatistics][OT.rSquare].to_s + end + reports = OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"], + "report/crossvalidation?crossvalidation_uris="+cv.crossvalidation_uri),{:subjectid => r.subjectid}) + if reports and reports.chomp.size>0 + lmo << "for more info see report: "+reports.split("\n")[0] + else + lmo << "for more info see report: not yet created for '"+cv.crossvalidation_uri+"'" + end + rescue => ex + LOGGER.warn "could not add cv "+cv.crossvalidation_uri+" : "+ex.message end end lmo << "" end end - - else + else lmo = [ "no prediction algortihm for model found, crossvalidation not possible" ] end r.qsar_robustness.lmo = lmo.to_html diff --git a/report/plot_factory.rb b/report/plot_factory.rb index a4e415a..b7c920a 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -354,7 +354,7 @@ module Reports c = roc_values[:confidence_values] p = roc_values[:predicted_values] a = roc_values[:actual_values] - raise "no prediction values for roc-plot" if p.size==0 + raise "no prediction values for confidence plot" if p.size==0 (0..p.size-2).each do |i| ((i+1)..p.size-1).each do |j| diff --git a/report/report_content.rb b/report/report_content.rb index ca04f25..cc4c13c 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -31,11 +31,11 @@ class Reports::ReportContent level = 0.90 test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, group_attribute, test_attribute, "paired_ttest", level ) - puts test_matrix.inspect + #puts test_matrix.inspect titles = test_matrix[:titles] matrix = test_matrix[:matrix] table = [] - puts titles.inspect + #puts titles.inspect table << [""] + titles titles.size.times do |i| table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") } @@ -47,10 +47,10 @@ class Reports::ReportContent end def add_predictions( validation_set, - validation_attributes=[], - section_title="Predictions", - section_text=nil, - table_title="Predictions") + validation_attributes=[], + section_title="Predictions", + section_text=nil, + table_title="Predictions") #PENING raise "validation attributes not implemented in get prediction array" if validation_attributes.size>0 @@ -109,7 +109,7 @@ class Reports::ReportContent if (search_for_existing_report_type) vals.size.times do |i| - puts i + #puts i if (i==0) vals[i] = [ "Reports" ] + vals[i] puts vals[i].inspect diff --git a/report/validation_access.rb b/report/validation_access.rb index e9b6e19..d0c3a1d 100755 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -25,7 +25,7 @@ class Reports::ValidationDB raise OpenTox::NotFoundError.new "crossvalidation with id "+cv_id.to_s+" not found" unless cv raise OpenTox::BadRequestError.new("crossvalidation with id '"+cv_id.to_s+"' not finished") unless cv.finished #res += Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s} - res += Validation::Validation.find( :crossvalidation_id => cv_id ).collect{|v| v.validation_uri.to_s } + res += Validation::Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|v| v.validation_uri.to_s } else res += [u.to_s] end @@ -35,7 +35,7 @@ class Reports::ValidationDB def init_validation(validation, uri, subjectid=nil) - raise OpenTox::BadRequestError.new "not a validation uri: "+uri.to_s unless uri =~ /.*\/[0-9]+/ + raise OpenTox::BadRequestError.new "not a validation uri: "+uri.to_s unless uri =~ /\/[0-9]+$/ validation_id = uri.split("/")[-1] raise OpenTox::BadRequestError.new "invalid validation id "+validation_id.to_s unless validation_id!=nil and (validation_id.to_i > 0 || validation_id.to_s=="0" ) @@ -56,6 +56,31 @@ class Reports::ValidationDB subset_props.each{ |prop| validation.send("#{prop.to_s}=".to_sym, subset[prop]) } if subset end end + + def init_validation_from_cv_statistics( validation, cv_uri, subjectid=nil ) + + raise OpenTox::BadRequestError.new "not a crossvalidation uri: "+cv_uri.to_s unless cv_uri.uri? and cv_uri =~ /crossvalidation.*\/[0-9]+$/ + cv_id = cv_uri.split("/")[-1] + raise OpenTox::NotAuthorizedError.new "Not authorized: GET "+cv_uri.to_s if + AA_SERVER and !OpenTox::Authorization.authorized?(cv_uri,"GET",subjectid) + cv = Validation::Crossvalidation.get(cv_id) + raise OpenTox::NotFoundError.new "crossvalidation with id "+crossvalidation_id.to_s+" not found" unless cv + raise OpenTox::BadRequestError.new "crossvalidation with id "+crossvalidation_id.to_s+" is not finished yet" unless cv.finished + v = Validation::Validation.from_cv_statistics(cv_id, subjectid) + (Validation::VAL_PROPS + Validation::VAL_CV_PROPS).each do |p| + validation.send("#{p.to_s}=".to_sym, v.send(p)) + end + {:classification_statistics => Validation::VAL_CLASS_PROPS, + :regression_statistics => Validation::VAL_REGR_PROPS}.each do |subset_name,subset_props| + subset = v.send(subset_name) + subset_props.each{ |prop| validation.send("#{prop.to_s}=".to_sym, subset[prop]) } if subset + end + #cv props + Validation::CROSS_VAL_PROPS.each do |p| + validation.send("#{p.to_s}=".to_sym, cv.send(p.to_s)) + end + validation.crossvalidation_uri = cv_uri + end def init_cv(validation) @@ -76,7 +101,8 @@ class Reports::ValidationDB def get_accept_values( validation, subjectid=nil ) # PENDING So far, one has to load the whole dataset to get the accept_value from ambit - d = OpenTox::Dataset.find( validation.test_target_dataset_uri, subjectid ) + d = Lib::DatasetCache.find( validation.test_target_dataset_uri, subjectid ) + raise "cannot get test target dataset for accept values, dataset: "+validation.test_target_dataset_uri.to_s unless d accept_values = d.features[validation.prediction_feature][OT.acceptValue] raise "cannot get accept values from dataset "+validation.test_target_dataset_uri.to_s+" for feature "+ validation.prediction_feature+":\n"+d.features[validation.prediction_feature].to_yaml unless accept_values!=nil diff --git a/report/validation_data.rb b/report/validation_data.rb index 42b179b..11fa737 100755 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -81,6 +81,12 @@ module Reports @subjectid = subjectid #raise "subjectid is nil" unless subjectid end + + def self.from_cv_statistics( cv_uri, subjectid = nil ) + v = ReportValidation.new(nil, subjectid) + @@validation_access.init_validation_from_cv_statistics(v, cv_uri, subjectid) + v + end # returns/creates predictions, cache to save rest-calls/computation time # @@ -409,17 +415,30 @@ module Reports #compute grouping grouping = Util.group(@validations, equal_attributes) #puts "groups "+grouping.size.to_s - - Lib::MergeObjects.register_merge_attributes( ReportValidation, - Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL) unless - Lib::MergeObjects.merge_attributes_registered?(ReportValidation) - - #merge - grouping.each do |g| - new_set.validations.push(g[0].clone_validation) - g[1..-1].each do |v| - new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v) + + if ( equal_attributes.include?(:crossvalidation_id) ) + # do not merge, use crossvalidation statistics + raise "statistics vs merging problem" if equal_attributes.size!=1 + grouping.each do |g| + new_set.validations << ReportValidation.from_cv_statistics(g[0].crossvalidation_uri) end + else + #merge + Lib::MergeObjects.register_merge_attributes( ReportValidation, + Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL) unless + Lib::MergeObjects.merge_attributes_registered?(ReportValidation) + grouping.each do |g| + new_set.validations << g[0].clone_validation + w = 1 + g[1..-1].each do |v| + new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v,w,1) + w+=1 + end + end + end + + new_set.validations.each do |v| + raise "not a validation "+v.class.to_s+" "+v.to_s unless v.is_a?(Reports::ReportValidation) end return new_set diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 4bcd07d..7db2a6a 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -3,8 +3,7 @@ require lib end -require 'lib/merge.rb' -#require 'lib/active_record_setup.rb' +require 'lib/dataset_cache.rb' require 'validation/validation_service.rb' get '/crossvalidation/?' do @@ -41,6 +40,8 @@ post '/crossvalidation/?' do cv = Validation::Crossvalidation.create cv_params cv.subjectid = @subjectid cv.perform_cv( params[:prediction_feature], params[:algorithm_params], task ) + # computation of stats is cheap as dataset are already loaded into the memory + Validation::Validation.from_cv_statistics( cv.id, @subjectid ) cv.crossvalidation_uri end return_task(task) @@ -108,33 +109,9 @@ get '/crossvalidation/:id' do end get '/crossvalidation/:id/statistics' do - LOGGER.info "get merged validation-result for crossvalidation with id "+params[:id].to_s -# begin - #crossvalidation = Validation::Crossvalidation.find(params[:id]) -# rescue ActiveRecord::RecordNotFound => ex -# raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found." -# end - #crossvalidation = Validation::Crossvalidation.find(params[:id]) - crossvalidation = Validation::Crossvalidation.get(params[:id]) - - raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found." unless crossvalidation - raise OpenTox::BadRequestError.new "Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished - - Lib::MergeObjects.register_merge_attributes( Validation::Validation, - Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL-[:date,:validation_uri,:crossvalidation_uri]) unless - Lib::MergeObjects.merge_attributes_registered?(Validation::Validation) - - #v = Lib::MergeObjects.merge_array_objects( Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) ) - # convert ohm:set into array, as ohm:set[0]=nil(!) - vals = Validation::Validation.find( :crossvalidation_id => params[:id] ).collect{|x| x} -# LOGGER.debug vals.collect{|v| v.validation_uri}.join("\n") -# LOGGER.debug vals.size -# LOGGER.debug vals.class - raise "could not load all validations for crossvalidation" if vals.include?(nil) - v = Lib::MergeObjects.merge_array_objects( vals ) - v.date = nil - #v.id = nil + LOGGER.info "get crossvalidation statistics for crossvalidation with id "+params[:id].to_s + v = Validation::Validation.from_cv_statistics( params[:id], @subjectid ) case request.env['HTTP_ACCEPT'].to_s when /text\/html/ related_links = @@ -187,7 +164,7 @@ get '/crossvalidation/:id/predictions' do raise OpenTox::BadRequestError.new "Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished content_type "application/x-yaml" - validations = Validation::Validation.find( :crossvalidation_id => params[:id] ) + validations = Validation::Validation.find( :crossvalidation_id => params[:id], :validation_type => "crossvalidation" ) p = Lib::OTPredictions.to_array( validations.collect{ |v| v.compute_validation_stats_with_model(nil, true) } ).to_yaml case request.env['HTTP_ACCEPT'].to_s diff --git a/validation/validation_format.rb b/validation/validation_format.rb index 6fdea61..23b1996 100755 --- a/validation/validation_format.rb +++ b/validation/validation_format.rb @@ -83,7 +83,7 @@ module Validation end v = [] #Validation.find( :all, :conditions => { :crossvalidation_id => self.id } ).each do |val| - Validation.find( :crossvalidation_id => self.id ).each do |val| + Validation.find( :crossvalidation_id => self.id, :validation_type => "crossvalidation" ).each do |val| v.push( val.validation_uri.to_s ) end h[:validation_uris] = v diff --git a/validation/validation_service.rb b/validation/validation_service.rb index dcfb8d7..99d8672 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -31,15 +31,49 @@ end module Validation class Validation - - # constructs a validation object, Rsets id und uri - #def initialize( params={} ) - #raise "do not set id manually" if params[:id] - #params[:finished] = false - #super params - #self.save! - #raise "internal error, validation-id not set "+to_yaml if self.id==nil - #end + + def self.from_cv_statistics( cv_id, subjectid=nil ) + v = Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation_statistics" ).first + unless v + crossvalidation = Crossvalidation.get(cv_id) + raise OpenTox::NotFoundError.new "Crossvalidation '#{cv_id}' not found." unless crossvalidation + raise OpenTox::BadRequestError.new "Crossvalidation '"+cv_id.to_s+"' not finished" unless crossvalidation.finished + + vals = Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|x| x} + feature_type = OpenTox::Model::Generic.new(vals.first.model_uri).feature_type(@subjectid) + test_dataset_uris = vals.collect{|v| v.test_dataset_uri} + test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri} + prediction_feature = vals.first.prediction_feature + prediction_dataset_uris = vals.collect{|v| v.prediction_dataset_uri} + predicted_variables = vals.collect{|v| nil} + prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, + prediction_dataset_uris, predicted_variables, @subjectid ) + + v = Validation.new + case feature_type + when "classification" + v.classification_statistics = prediction.compute_stats + when "regression" + v.regression_statistics = prediction.compute_stats + end + v.update :num_instances => prediction.num_instances, + :num_without_class => prediction.num_without_class, + :percent_without_class => prediction.percent_without_class, + :num_unpredicted => prediction.num_unpredicted, + :percent_unpredicted => prediction.percent_unpredicted, + :finished => true + (VAL_PROPS_GENERAL-[:validation_uri]).each do |p| + v.send("#{p.to_s}=".to_sym, vals.collect{ |vv| vv.send(p) }.uniq.join(",")) + end + v.date = crossvalidation.date + v.validation_type = "crossvalidation_statistics" + v.crossvalidation_id = crossvalidation.id + v.crossvalidation_fold = vals.collect{ |vv| vv.crossvalidation_fold }.uniq.join(",") + v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(",") + v.save + end + v + end # deletes a validation # PENDING: model and referenced datasets are deleted as well, keep it that way? @@ -238,21 +272,7 @@ module Validation class Crossvalidation - # constructs a crossvalidation, id and uri are set - #def initialize( params={} ) - # - # raise "do not set id manually" if params[:id] - # params[:num_folds] = 10 if params[:num_folds]==nil - # params[:random_seed] = 1 if params[:random_seed]==nil - # params[:stratified] = false if params[:stratified]==nil - # params[:finished] = false - # super params - # self.save! - # raise "internal error, crossvalidation-id not set" if self.id==nil - #end - def perform_cv ( prediction_feature, algorithm_params=nil, task=nil ) - create_cv_datasets( prediction_feature, OpenTox::SubTask.create(task, 0, 33) ) perform_cv_validations( algorithm_params, OpenTox::SubTask.create(task, 33, 100) ) end @@ -324,7 +344,7 @@ module Validation cvs.each do |cv| next if AA_SERVER and !OpenTox::Authorization.authorized?(cv.crossvalidation_uri,"GET",self.subjectid) tmp_val = [] - Validation.find( :crossvalidation_id => cv.id ).each do |v| + Validation.find( :crossvalidation_id => cv.id, :validation_type => "crossvalidation" ).each do |v| break unless v.prediction_feature == prediction_feature and OpenTox::Dataset.exist?(v.training_dataset_uri,self.subjectid) and @@ -353,7 +373,7 @@ module Validation # stores uris in validation objects def create_new_cv_datasets( prediction_feature, task = nil ) LOGGER.debug "creating datasets for crossvalidation" - orig_dataset = OpenTox::Dataset.find(self.dataset_uri,self.subjectid) + orig_dataset = Lib::DatasetCache.find(self.dataset_uri,self.subjectid) raise OpenTox::NotFoundError.new "Dataset not found: "+self.dataset_uri.to_s unless orig_dataset shuffled_compounds = orig_dataset.compounds.shuffle( self.random_seed ) @@ -465,7 +485,7 @@ module Validation random_seed=1 unless random_seed - orig_dataset = OpenTox::Dataset.find orig_dataset_uri,subjectid + orig_dataset = Lib::DatasetCache.find orig_dataset_uri,subjectid orig_dataset.load_all raise OpenTox::NotFoundError.new "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset if prediction_feature @@ -530,7 +550,7 @@ module Validation task.progress(100) if task if ENV['RACK_ENV'] =~ /test|debug/ - training_dataset = OpenTox::Dataset.find result[:training_dataset_uri],subjectid + training_dataset = Lib::DatasetCache.find result[:training_dataset_uri],subjectid raise OpenTox::NotFoundError.new "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless training_dataset training_dataset.load_all value_count = 0 @@ -539,7 +559,7 @@ module Validation end raise "training compounds error" unless value_count==training_compounds.size raise OpenTox::NotFoundError.new "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless - OpenTox::Dataset.find result[:test_dataset_uri], subjectid + Lib::DatasetCache.find result[:test_dataset_uri], subjectid end LOGGER.debug "bootstrapping done, training dataset: '"+result[:training_dataset_uri].to_s+"', test dataset: '"+result[:test_dataset_uri].to_s+"'" @@ -554,7 +574,7 @@ module Validation random_seed=1 unless random_seed random_seed = random_seed.to_i - orig_dataset = OpenTox::Dataset.find orig_dataset_uri, subjectid + orig_dataset = Lib::DatasetCache.find orig_dataset_uri, subjectid orig_dataset.load_all subjectid raise OpenTox::NotFoundError.new "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset raise OpenTox::NotFoundError.new "Split ratio invalid: "+split_ratio.to_s unless split_ratio and split_ratio=split_ratio.to_f @@ -597,7 +617,7 @@ module Validation subjectid ).uri task.progress(66) if task -# d = OpenTox::Dataset.find(result[:training_dataset_uri]) +# d = Lib::DatasetCache.find(result[:training_dataset_uri]) # d.data_entries.values.each do |v| # puts v.inspect # puts v.values[0].to_s+" "+v.values[0].class.to_s @@ -617,8 +637,8 @@ module Validation if ENV['RACK_ENV'] =~ /test|debug/ raise OpenTox::NotFoundError.new "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless - OpenTox::Dataset.find(result[:training_dataset_uri],subjectid) - test_data = OpenTox::Dataset.find result[:test_dataset_uri],subjectid + Lib::DatasetCache.find(result[:training_dataset_uri],subjectid) + test_data = Lib::DatasetCache.find result[:test_dataset_uri],subjectid raise OpenTox::NotFoundError.new "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless test_data test_data.load_compounds subjectid raise "Test dataset num coumpounds != "+(compounds.size-split-1).to_s+", instead: "+ |