From 324e7f2dc7c8417fd5af0a084f06ecc92de41d48 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 2 Apr 2012 16:11:28 +0200 Subject: new stratified type super added to crossvalidation and traning-test-split validation, add some more metadata to crossvaldiation, add validation_uri to predictions in crossvaldiation-report --- lib/ot_predictions.rb | 21 ++-- lib/validation_db.rb | 18 ++- report/report_content.rb | 11 +- report/report_factory.rb | 4 +- validation/validation_application.rb | 46 +++++--- validation/validation_service.rb | 217 ++++++++++++++++------------------- 6 files changed, 163 insertions(+), 154 deletions(-) diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index 3be845b..2752fcc 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -35,7 +35,7 @@ module Lib OTPredictions.to_array( [self] ) end - def self.to_array( predictions, add_pic=false, format=false ) + def self.to_array( predictions, add_pic=false, format=false, validation_uris=nil ) confidence_available = false predictions.each do |p| @@ -43,7 +43,10 @@ module Lib end res = [] conf_column = nil + count = 0 predictions.each do |p| + v_uris = validation_uris[count] if validation_uris + count += 1 (0..p.num_instances-1).each do |i| a = [] @@ -75,6 +78,9 @@ module Lib conf_column = a.size if conf_column==nil a << p.confidence_value(i) end + if validation_uris + a << v_uris[i] + end a << p.identifier(i) res << a end @@ -90,12 +96,13 @@ module Lib end end header = [] - header << "compound" if add_pic - header << "actual value" - header << "predicted value" - header << "classification" if predictions[0].feature_type=="classification" - header << "confidence value" if predictions[0].confidence_values_available? - header << "compound-uri" + header << "Compound" if add_pic + header << "Actual value" + header << "Predicted value" + header << "Classification" if predictions[0].feature_type=="classification" + header << "Confidence value" if predictions[0].confidence_values_available? + header << "Validation URI" if validation_uris + header << "Compound URI" res.insert(0, header) return res diff --git a/lib/validation_db.rb b/lib/validation_db.rb index 7d83966..086853e 100755 --- a/lib/validation_db.rb +++ b/lib/validation_db.rb @@ -6,8 +6,9 @@ require "lib/merge.rb" module Validation - VAL_PROPS_GENERAL = [ :validation_uri, :validation_type, :model_uri, :algorithm_uri, :training_dataset_uri, :prediction_feature, - :test_dataset_uri, :test_target_dataset_uri, :prediction_dataset_uri, :date ] + VAL_PROPS_GENERAL = [ :validation_uri, :validation_type, :model_uri, :algorithm_uri, :algorithm_params, + :training_dataset_uri, :prediction_feature, :test_dataset_uri, :test_target_dataset_uri, + :prediction_dataset_uri, :date ] VAL_PROPS_SUM = [ :num_instances, :num_without_class, :num_unpredicted ] VAL_PROPS_AVG = [:real_runtime, :percent_without_class, :percent_unpredicted ] VAL_PROPS = VAL_PROPS_GENERAL + VAL_PROPS_SUM + VAL_PROPS_AVG @@ -41,7 +42,8 @@ module Validation :weighted_mean_absolute_error, :weighted_root_mean_squared_error, :concordance_correlation_coefficient ] CROSS_VAL_PROPS = [:dataset_uri, :num_folds, :stratified, :random_seed] - CROSS_VAL_PROPS_REDUNDANT = [:crossvalidation_uri, :algorithm_uri, :date] + CROSS_VAL_PROPS + CROSS_VAL_PROPS_REDUNDANT = [:crossvalidation_uri, :algorithm_uri, :algorithm_params, + :prediction_feature, :date] + CROSS_VAL_PROPS ALL_PROPS = VAL_PROPS + VAL_CV_PROPS + VAL_CLASS_PROPS + VAL_REGR_PROPS + CROSS_VAL_PROPS @@ -55,6 +57,7 @@ module Validation attribute :validation_type attribute :model_uri attribute :algorithm_uri + attribute :algorithm_params attribute :training_dataset_uri attribute :test_target_dataset_uri attribute :test_dataset_uri @@ -77,6 +80,11 @@ module Validation index :model_uri index :validation_type index :crossvalidation_id + index :algorithm_uri + index :algorithm_params + index :prediction_feature + index :training_dataset_uri + index :test_dataset_uri attr_accessor :subjectid @@ -141,6 +149,8 @@ module Validation class Crossvalidation < Ohm::Model attribute :algorithm_uri + attribute :algorithm_params + attribute :prediction_feature attribute :dataset_uri attribute :date attribute :num_folds @@ -152,6 +162,8 @@ module Validation attr_accessor :subjectid index :algorithm_uri + index :algorithm_params + index :prediction_feature index :dataset_uri index :num_folds index :random_seed diff --git a/report/report_content.rb b/report/report_content.rb index 80473c5..033b367 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -63,20 +63,17 @@ class Reports::ReportContent end end - def add_predictions( validation_set, - validation_attributes=[], + def add_predictions( validation_set, + add_validation_uris, section_title="Predictions", section_text=nil, table_title="Predictions") - - #PENING - raise "validation attributes not implemented in get prediction array" if validation_attributes.size>0 - section_table = @xml_report.add_section(@current_section, section_title) if validation_set.validations[0].get_predictions @xml_report.add_paragraph(section_table, section_text) if section_text + v_uris = validation_set.validations.collect{|v| Array.new(v.num_instances.to_i,v.validation_uri)} if add_validation_uris @xml_report.add_table(section_table, table_title, Lib::OTPredictions.to_array(validation_set.validations.collect{|v| v.get_predictions}, - true, true)) + true, true, v_uris)) else @xml_report.add_paragraph(section_table, "No prediction info available.") end diff --git a/report/report_factory.rb b/report/report_factory.rb index f51b999..f73ffd9 100755 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -124,7 +124,7 @@ module Reports::ReportFactory report.end_section report.add_result(validation_set, Validation::ALL_PROPS, "All Results", "All Results") - report.add_predictions( validation_set ) + report.add_predictions( validation_set, false ) task.progress(100) if task report end @@ -200,7 +200,7 @@ module Reports::ReportFactory report.add_result(validation_set, Validation::ALL_PROPS, "All Results", "All Results") if (cv_set.unique_value(:num_folds).to_i < cv_set.unique_value(:num_instances).to_i) - report.add_predictions( validation_set ) #, [:crossvalidation_fold] ) + report.add_predictions( validation_set, true ) task.progress(100) if task report end diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 60fc7df..1bc55f6 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -6,8 +6,16 @@ end require 'lib/dataset_cache.rb' require 'validation/validation_service.rb' +helpers do + def check_stratified(params) + params[:stratified] = "false" unless params[:stratified] + raise OpenTox::BadRequestError.new "stratified != true|false|super, is #{params[:stratified]}" unless + params[:stratified]=~/true|false|super/ + end +end + get '/crossvalidation/?' do - LOGGER.info "list all crossvalidations" + LOGGER.info "list all crossvalidations "+params.inspect model_uri = params.delete("model") || params.delete("model_uri") if model_uri model = OpenTox::Model::Generic.find(model_uri, @subjectid) @@ -46,17 +54,20 @@ post '/crossvalidation/?' do raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0 raise OpenTox::BadRequestError.new "illegal param-value num_folds: '"+params[:num_folds].to_s+"', must be integer >1" unless params[:num_folds]==nil or params[:num_folds].to_i>1 - + check_stratified(params) + task = OpenTox::Task.create( "Perform crossvalidation", url_for("/crossvalidation", :full) ) do |task| #, params cv_params = { :dataset_uri => params[:dataset_uri], :algorithm_uri => params[:algorithm_uri], + :algorithm_params => params[:algorithm_params], + :prediction_feature => params[:prediction_feature], + :stratified => params[:stratified], :loo => "false", :subjectid => @subjectid } [ :num_folds, :random_seed ].each{ |sym| cv_params[sym] = params[sym] if params[sym] } - cv_params[:stratified] = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] cv = Validation::Crossvalidation.create cv_params cv.subjectid = @subjectid - cv.perform_cv( params[:prediction_feature], params[:algorithm_params], OpenTox::SubTask.create(task,0,95)) + cv.perform_cv( OpenTox::SubTask.create(task,0,95) ) # computation of stats is cheap as dataset are already loaded into the memory Validation::Validation.from_cv_statistics( cv.id, @subjectid, OpenTox::SubTask.create(task,95,100) ) cv.crossvalidation_uri @@ -87,14 +98,16 @@ post '/crossvalidation/loo/?' do raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri].to_s.size>0 raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0 raise OpenTox::BadRequestError.new "illegal param: num_folds, stratified, random_seed not allowed for loo-crossvalidation" if params[:num_folds] or - params[:stratifed] or params[:random_seed] + params[:stratified] or params[:random_seed] task = OpenTox::Task.create( "Perform loo-crossvalidation", url_for("/crossvalidation/loo", :full) ) do |task| #, params - cv_params = { :dataset_uri => params[:dataset_uri], + cv_params = { :dataset_uri => params[:dataset_uri], + :algorithm_params => params[:algorithm_params], + :prediction_feature => params[:prediction_feature], :algorithm_uri => params[:algorithm_uri], :loo => "true" } cv = Validation::Crossvalidation.create cv_params cv.subjectid = @subjectid - cv.perform_cv( params[:prediction_feature], params[:algorithm_params], OpenTox::SubTask.create(task,0,95)) + cv.perform_cv( OpenTox::SubTask.create(task,0,95)) # computation of stats is cheap as dataset are already loaded into the memory Validation::Validation.from_cv_statistics( cv.id, @subjectid, OpenTox::SubTask.create(task,95,100) ) cv.clean_loo_files( !(params[:algorithm_params] && params[:algorithm_params] =~ /feature_dataset_uri/) ) @@ -344,12 +357,13 @@ post '/training_test_validation/?' do task = OpenTox::Task.create( "Perform training-test-validation", url_for("/", :full) ) do |task| #, params v = Validation::Validation.create :validation_type => "training_test_validation", :algorithm_uri => params[:algorithm_uri], + :algorithm_params => params[:algorithm_params], :training_dataset_uri => params[:training_dataset_uri], :test_dataset_uri => params[:test_dataset_uri], :test_target_dataset_uri => params[:test_target_dataset_uri], :prediction_feature => params[:prediction_feature] v.subjectid = @subjectid - v.validate_algorithm( params[:algorithm_params], task ) + v.validate_algorithm( task ) v.validation_uri end return_task(task) @@ -403,10 +417,11 @@ post '/bootstrapping' do :test_target_dataset_uri => params[:dataset_uri], :prediction_feature => params[:prediction_feature], :algorithm_uri => params[:algorithm_uri], + :algorithm_params => params[:algorithm_params], :training_dataset_uri => params[:training_dataset_uri], :test_dataset_uri => params[:test_dataset_uri] v.subjectid = @subjectid - v.validate_algorithm( params[:algorithm_params], OpenTox::SubTask.create(task,33,100)) + v.validate_algorithm( OpenTox::SubTask.create(task,33,100)) v.validation_uri end return_task(task) @@ -453,18 +468,19 @@ post '/training_test_split' do raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri].to_s.size>0 raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri].to_s.size>0 raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0 + check_stratified(params) task = OpenTox::Task.create( "Perform training test split validation", url_for("/training_test_split", :full) ) do |task| #, params - strat = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] params.merge!( Validation::Util.train_test_dataset_split(params[:dataset_uri], params[:prediction_feature], - @subjectid, strat, params[:split_ratio], params[:random_seed], OpenTox::SubTask.create(task,0,33))) + @subjectid, params[:stratified], params[:split_ratio], params[:random_seed], OpenTox::SubTask.create(task,0,33))) v = Validation::Validation.create :validation_type => "training_test_split", :training_dataset_uri => params[:training_dataset_uri], :test_dataset_uri => params[:test_dataset_uri], :test_target_dataset_uri => params[:dataset_uri], :prediction_feature => params[:prediction_feature], - :algorithm_uri => params[:algorithm_uri] + :algorithm_uri => params[:algorithm_uri], + :algorithm_params => params[:algorithm_params] v.subjectid = @subjectid - v.validate_algorithm( params[:algorithm_params], OpenTox::SubTask.create(task,33,100)) + v.validate_algorithm( OpenTox::SubTask.create(task,33,100)) v.validation_uri end return_task(task) @@ -546,10 +562,10 @@ end post '/plain_training_test_split' do LOGGER.info "creating pure training test split "+params.inspect raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] + check_stratified(params) task = OpenTox::Task.create( "Create data-split", url_for("/plain_training_test_split", :full) ) do |task| - strat = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] result = Validation::Util.train_test_dataset_split(params[:dataset_uri], params[:prediction_feature], @subjectid, - strat, params[:split_ratio], params[:random_seed]) + params[:stratified], params[:split_ratio], params[:random_seed], task) content_type "text/uri-list" result[:training_dataset_uri]+"\n"+result[:test_dataset_uri]+"\n" end diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 686a287..c433161 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -111,7 +111,7 @@ module Validation end # validates an algorithm by building a model and validating this model - def validate_algorithm( algorithm_params=nil, task=nil ) + def validate_algorithm( task=nil ) raise "validation_type missing" unless self.validation_type raise OpenTox::BadRequestError.new "no algorithm uri: '"+self.algorithm_uri.to_s+"'" if self.algorithm_uri==nil or self.algorithm_uri.to_s.size<1 @@ -301,9 +301,9 @@ module Validation class Crossvalidation - def perform_cv ( prediction_feature, algorithm_params=nil, task=nil ) - create_cv_datasets( prediction_feature, OpenTox::SubTask.create(task, 0, 33) ) - perform_cv_validations( algorithm_params, OpenTox::SubTask.create(task, 33, 100) ) + def perform_cv ( task=nil ) + create_cv_datasets( OpenTox::SubTask.create(task, 0, 33) ) + perform_cv_validations( OpenTox::SubTask.create(task, 33, 100) ) end def clean_loo_files( delete_feature_datasets ) @@ -349,27 +349,27 @@ module Validation end # creates the cv folds - def create_cv_datasets( prediction_feature, task=nil ) + def create_cv_datasets( task=nil ) if self.loo=="true" orig_dataset = Lib::DatasetCache.find(self.dataset_uri,self.subjectid) self.num_folds = orig_dataset.compounds.size self.random_seed = 0 - self.stratified = false + self.stratified = "false" else self.random_seed = 1 unless self.random_seed self.num_folds = 10 unless self.num_folds - self.stratified = false unless self.stratified + self.stratified = "false" unless self.stratified end - if copy_cv_datasets( prediction_feature ) + if copy_cv_datasets() # dataset folds of a previous crossvalidaiton could be used task.progress(100) if task else - create_new_cv_datasets( prediction_feature, task ) + create_new_cv_datasets( task ) end end # executes the cross-validation (build models and validates them) - def perform_cv_validations( algorithm_params, task=nil ) + def perform_cv_validations( task=nil ) LOGGER.debug "perform cv validations "+algorithm_params.inspect i = 0 @@ -377,8 +377,7 @@ module Validation @tmp_validations.each do | val | validation = Validation.create val validation.subjectid = self.subjectid - validation.validate_algorithm( algorithm_params, - OpenTox::SubTask.create(task, i * task_step, ( i + 1 ) * task_step) ) + validation.validate_algorithm( OpenTox::SubTask.create(task, i * task_step, ( i + 1 ) * task_step) ) raise "validation '"+validation.validation_uri+"' for crossvaldation could not be finished" unless validation.finished i += 1 @@ -395,14 +394,17 @@ module Validation private # copies datasets from an older crossvalidation on the same dataset and the same folds # returns true if successfull, false otherwise - def copy_cv_datasets( prediction_feature ) + def copy_cv_datasets( ) + # for downwards compatibilty: search prediction_feature=nil is ok cvs = Crossvalidation.find( { :dataset_uri => self.dataset_uri, :num_folds => self.num_folds, :stratified => self.stratified, :random_seed => self.random_seed, :loo => self.loo, - :finished => true} ).reject{ |cv| cv.id == self.id } + :finished => true} ).reject{ |cv| (cv.id == self.id || + (cv.prediction_feature && + cv.prediction_feature != self.prediction_feature)) } cvs.each do |cv| next if AA_SERVER and !OpenTox::Authorization.authorized?(cv.crossvalidation_uri,"GET",self.subjectid) tmp_val = [] @@ -420,7 +422,8 @@ module Validation :crossvalidation_id => self.id, :crossvalidation_fold => v.crossvalidation_fold, :prediction_feature => prediction_feature, - :algorithm_uri => self.algorithm_uri } + :algorithm_uri => self.algorithm_uri, + :algorithm_params => self.algorithm_params } end if tmp_val.size == self.num_folds.to_i @tmp_validations = tmp_val @@ -433,111 +436,78 @@ module Validation # creates cv folds (training and testdatasets) # stores uris in validation objects - def create_new_cv_datasets( prediction_feature, task = nil ) + def create_new_cv_datasets( task = nil ) LOGGER.debug "creating datasets for crossvalidation" orig_dataset = Lib::DatasetCache.find(self.dataset_uri,self.subjectid) raise OpenTox::NotFoundError.new "Dataset not found: "+self.dataset_uri.to_s unless orig_dataset - if self.loo=="true" - shuffled_compounds = orig_dataset.compounds - else - shuffled_compounds = orig_dataset.compounds.shuffle( self.random_seed ) - end + train_dataset_uris = [] + test_dataset_uris = [] - unless self.stratified + meta = { DC.creator => self.crossvalidation_uri } + case stratified + when "false" + if self.loo=="true" + shuffled_compounds = orig_dataset.compounds + else + shuffled_compounds = orig_dataset.compounds.shuffle( self.random_seed ) + end split_compounds = shuffled_compounds.chunk( self.num_folds.to_i ) - else - class_compounds = {} # "inactive" => compounds[], "active" => compounds[] .. - accept_values = orig_dataset.accept_values(prediction_feature) - raise OpenTox::BadRequestError.new("cannot apply stratification (not implemented for regression), acceptValue missing for prediction-feature '"+ - prediction_feature.to_s+"' in dataset '"+dataset_uri.to_s+"'") unless accept_values and accept_values.size>0 - accept_values.each do |value| - class_compounds[value] = [] - shuffled_compounds.each do |c| - #PENDING accept values are type string, data_entries may be boolean - class_compounds[value] << c if orig_dataset.data_entries[c][prediction_feature].collect{|v| v.to_s}.include?(value) - end - end - LOGGER.debug "stratified cv: different class values: "+class_compounds.keys.join(", ") - LOGGER.debug "stratified cv: num instances for each class value: "+class_compounds.values.collect{|c| c.size}.join(", ") - - split_class_compounds = [] # inactive_compounds[fold_i][], active_compounds[fold_i][], .. - class_compounds.values.each do |compounds| - split_class_compounds << compounds.chunk( self.num_folds.to_i ) - end - LOGGER.debug "stratified cv: splits for class values: "+split_class_compounds.collect{ |c| c.collect{ |cc| cc.size }.join("/") }.join(", ") - - # we cannot just merge the splits of the different class_values of each fold - # this could lead to folds, which sizes differ for more than 1 compound - split_compounds = [] - split_class_compounds.each do |split_comp| - # step 1: sort current split in ascending order - split_comp.sort!{|x,y| x.size <=> y.size } - # step 2: add splits - (0..self.num_folds.to_i-1).each do |i| - unless split_compounds[i] - split_compounds[i] = split_comp[i] + LOGGER.debug "cv: num instances for each fold: "+split_compounds.collect{|c| c.size}.join(", ") + + self.num_folds.to_i.times do |n| + test_compounds = [] + train_compounds = [] + self.num_folds.to_i.times do |nn| + compounds = split_compounds[nn] + if n == nn + compounds.each{ |compound| test_compounds << compound} else - split_compounds[i] += split_comp[i] - end + compounds.each{ |compound| train_compounds << compound} + end end - # step 3: sort (total) split in descending order - split_compounds.sort!{|x,y| y.size <=> x.size } + raise "internal error, num test compounds not correct,"+ + " is '#{test_compounds.size}', should be '#{(shuffled_compounds.size/self.num_folds.to_i)}'" unless + (shuffled_compounds.size/self.num_folds.to_i - test_compounds.size).abs <= 1 + raise "internal error, num train compounds not correct, should be '"+(shuffled_compounds.size-test_compounds.size).to_s+ + "', is '"+train_compounds.size.to_s+"'" unless shuffled_compounds.size - test_compounds.size == train_compounds.size + datasetname = 'dataset fold '+(n+1).to_s+' of '+self.num_folds.to_s + meta[DC.title] = "training "+datasetname + LOGGER.debug "training set: "+datasetname+"_train, compounds: "+train_compounds.size.to_s + train_dataset_uri = orig_dataset.split( train_compounds, orig_dataset.features.keys, + meta, self.subjectid ).uri + train_dataset_uris << train_dataset_uri + meta[DC.title] = "test "+datasetname + LOGGER.debug "test set: "+datasetname+"_test, compounds: "+test_compounds.size.to_s + test_features = orig_dataset.features.keys.dclone - [self.prediction_feature] + test_dataset_uri = orig_dataset.split( test_compounds, test_features, + meta, self.subjectid ).uri + test_dataset_uris << test_dataset_uri + end + when /true|super/ + if stratified=="true" + features = [ self.prediction_feature ] + else + features = nil end + train_datasets, test_datasets = stratified_k_fold_split(orig_dataset,meta, + "NA",self.num_folds.to_i,@subjectid,self.random_seed, features) + train_dataset_uris = test_datasets.collect{|d| d.uri} + test_dataset_uris = test_datasets.collect{|d| d.uri} + else + raise OpenTox::BadRequestError.new end - LOGGER.debug "cv: num instances for each fold: "+split_compounds.collect{|c| c.size}.join(", ") - - test_features = orig_dataset.features.keys.dclone - [prediction_feature] @tmp_validations = [] - - (1..self.num_folds.to_i).each do |n| - - datasetname = 'cv'+self.id.to_s + - #'_d'+orig_dataset.name.to_s + - '_f'+n.to_s+'of'+self.num_folds.to_s+ - '_r'+self.random_seed.to_s+ - '_s'+self.stratified.to_s - source = self.crossvalidation_uri - - test_compounds = [] - train_compounds = [] - - (1..self.num_folds.to_i).each do |nn| - compounds = split_compounds.at(nn-1) - - if n == nn - compounds.each{ |compound| test_compounds.push(compound)} - else - compounds.each{ |compound| train_compounds.push(compound)} - end - end - - raise "internal error, num test compounds not correct" unless (shuffled_compounds.size/self.num_folds.to_i - test_compounds.size).abs <= 1 - raise "internal error, num train compounds not correct, should be '"+(shuffled_compounds.size-test_compounds.size).to_s+ - "', is '"+train_compounds.size.to_s+"'" unless shuffled_compounds.size - test_compounds.size == train_compounds.size - - LOGGER.debug "training set: "+datasetname+"_train, compounds: "+train_compounds.size.to_s - #train_dataset_uri = orig_dataset.create_new_dataset( train_compounds, orig_dataset.features, datasetname + '_train', source ) - train_dataset_uri = orig_dataset.split( train_compounds, orig_dataset.features.keys, - { DC.title => datasetname + '_train', DC.creator => source }, self.subjectid ).uri - - LOGGER.debug "test set: "+datasetname+"_test, compounds: "+test_compounds.size.to_s - #test_dataset_uri = orig_dataset.create_new_dataset( test_compounds, test_features, datasetname + '_test', source ) - test_dataset_uri = orig_dataset.split( test_compounds, test_features, - { DC.title => datasetname + '_test', DC.creator => source }, self.subjectid ).uri - - #make sure self.id is set - #self.save if self.new? + self.num_folds.to_i.times do |n| tmp_validation = { :validation_type => "crossvalidation", - :training_dataset_uri => train_dataset_uri, - :test_dataset_uri => test_dataset_uri, + :training_dataset_uri => train_dataset_uris[n], + :test_dataset_uri => test_dataset_uris[n], :test_target_dataset_uri => self.dataset_uri, - :crossvalidation_id => self.id, :crossvalidation_fold => n, - :prediction_feature => prediction_feature, + :crossvalidation_id => self.id, :crossvalidation_fold => (n+1), + :prediction_feature => self.prediction_feature, :algorithm_uri => self.algorithm_uri } @tmp_validations << tmp_validation - task.progress( n / self.num_folds.to_f * 100 ) if task end end @@ -636,7 +606,7 @@ module Validation # splits a dataset into test and training dataset # returns map with training_dataset_uri and test_dataset_uri - def self.train_test_dataset_split( orig_dataset_uri, prediction_feature, subjectid, stratified=false, split_ratio=nil, random_seed=nil, task=nil ) + def self.train_test_dataset_split( orig_dataset_uri, prediction_feature, subjectid, stratified="false", split_ratio=nil, random_seed=nil, task=nil ) split_ratio=0.67 unless split_ratio split_ratio = split_ratio.to_f random_seed=1 unless random_seed @@ -652,15 +622,25 @@ module Validation "' not found in dataset, features are: \n"+ orig_dataset.features.keys.inspect unless orig_dataset.features.include?(prediction_feature) else - LOGGER.warn "no prediciton feature given, all features included in test dataset" + LOGGER.warn "no prediciton feature given, all features will be included in test dataset" end - if stratified + meta = { DC.creator => $url_provider.url_for('/training_test_split',:full) } + + case stratified + when /true|super/ + if stratified=="true" + raise OpenTox::BadRequestError.new "prediction feature required for stratified splits" unless prediction_feature + features = [prediction_feature] + else + LOGGER.warn "prediction feature is ignored for super-stratified splits" if prediction_feature + features = nil + end r_util = OpenTox::RUtil.new - split_sets = r_util.stratified_split( orig_dataset, "NA", df, split_ratio, random_seed ) + train, test = r_util.stratified_split( orig_dataset, meta, "NA", split_ratio, @subjectid, random_seed, features ) r_util.quit_r - result = {:training_dataset_uri => split_sets[0], :test_dataset_uri => split_sets[1]} - else + result = {:training_dataset_uri => train.uri, :test_dataset_uri => test.uri} + when "false" compounds = orig_dataset.compounds raise OpenTox::BadRequestError.new "Cannot split datset, num compounds in dataset < 2 ("+compounds.size.to_s+")" if compounds.size<2 split = (compounds.size*split_ratio).to_i @@ -674,22 +654,18 @@ module Validation test_compounds = compounds[(split+1)..-1] task.progress(33) if task + meta[DC.title] = "Training dataset split of "+orig_dataset.uri result = {} result[:training_dataset_uri] = orig_dataset.split( training_compounds, - orig_dataset.features.keys, - { DC.title => "Training dataset split of "+orig_dataset.title.to_s, - DC.creator => $url_provider.url_for('/training_test_split',:full) }, - subjectid ).uri + orig_dataset.features.keys, meta, subjectid ).uri task.progress(66) if task + meta[DC.title] = "Test dataset split of "+orig_dataset.uri result[:test_dataset_uri] = orig_dataset.split( test_compounds, - orig_dataset.features.keys.dclone - [prediction_feature], - { DC.title => "Test dataset split of "+orig_dataset.title.to_s, - DC.creator => $url_provider.url_for('/training_test_split',:full) }, - subjectid ).uri + orig_dataset.features.keys.dclone - [prediction_feature], meta, subjectid ).uri task.progress(100) if task - if !stratified and ENV['RACK_ENV'] =~ /test|debug/ + if ENV['RACK_ENV'] =~ /test|debug/ raise OpenTox::NotFoundError.new "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless Lib::DatasetCache.find(result[:training_dataset_uri],subjectid) test_data = Lib::DatasetCache.find result[:test_dataset_uri],subjectid @@ -698,8 +674,9 @@ module Validation raise "Test dataset num coumpounds != "+(compounds.size-split-1).to_s+", instead: "+ test_data.compounds.size.to_s+"\n"+test_data.to_yaml unless test_data.compounds.size==(compounds.size-1-split) end - LOGGER.debug "split done, training dataset: '"+result[:training_dataset_uri].to_s+"', test dataset: '"+result[:test_dataset_uri].to_s+"'" + else + raise OpenTox::BadRequestError.new "stratified != false|true|super, is #{stratified}" end result end -- cgit v1.2.3