From 9d17895ab9e8cd31e0f32e8e622e13612ea5ff77 Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Fri, 12 Oct 2018 21:58:36 +0200 Subject: validation statistic fixes --- lib/leave-one-out-validation.rb | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) (limited to 'lib/leave-one-out-validation.rb') diff --git a/lib/leave-one-out-validation.rb b/lib/leave-one-out-validation.rb index c33c92b..b0905b8 100644 --- a/lib/leave-one-out-validation.rb +++ b/lib/leave-one-out-validation.rb @@ -12,7 +12,7 @@ module OpenTox bad_request_error "Cannot create leave one out validation for models with supervised feature selection. Please use crossvalidation instead." if model.algorithms[:feature_selection] $logger.debug "#{model.name}: LOO validation started" t = Time.now - model.training_dataset.features.first.nominal? ? klass = ClassificationLeaveOneOut : klass = RegressionLeaveOneOut + model.training_dataset.features.collect{|f| f.class}.include?(NominalBioActivity) ? klass = ClassificationLeaveOneOut : klass = RegressionLeaveOneOut loo = klass.new :model_id => model.id predictions = model.predict model.training_dataset.substances predictions.each{|cid,p| p.delete(:neighbors)} @@ -40,25 +40,27 @@ module OpenTox class ClassificationLeaveOneOut < LeaveOneOut include ClassificationStatistics field :accept_values, type: Array - field :confusion_matrix, type: Array, default: [] - field :weighted_confusion_matrix, type: Array, default: [] - field :accuracy, type: Float - field :weighted_accuracy, type: Float - field :true_rate, type: Hash, default: {} - field :predictivity, type: Hash, default: {} - field :confidence_plot_id, type: BSON::ObjectId + field :confusion_matrix, type: Hash + field :weighted_confusion_matrix, type: Hash + field :accuracy, type: Hash + field :weighted_accuracy, type: Hash + field :true_rate, type: Hash + field :predictivity, type: Hash + field :nr_predictions, type: Hash + field :probability_plot_id, type: BSON::ObjectId end # Leave one out validation for regression models class RegressionLeaveOneOut < LeaveOneOut include RegressionStatistics - field :rmse, type: Float, default: 0 - field :mae, type: Float, default: 0 - field :r_squared, type: Float - field :within_prediction_interval, type: Integer, default:0 - field :out_of_prediction_interval, type: Integer, default:0 - field :correlation_plot_id, type: BSON::ObjectId + field :rmse, type: Hash + field :mae, type: Hash + field :r_squared, type: Hash + field :within_prediction_interval, type: Hash + field :out_of_prediction_interval, type: Hash + field :nr_predictions, type: Hash field :warnings, type: Array + field :correlation_plot_id, type: BSON::ObjectId end end -- cgit v1.2.3 From 3a9c9332b660d35720ad4fa1f55ee0883e53aecd Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Fri, 2 Nov 2018 20:34:44 +0100 Subject: warnings fixed, cleanup --- lib/leave-one-out-validation.rb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'lib/leave-one-out-validation.rb') diff --git a/lib/leave-one-out-validation.rb b/lib/leave-one-out-validation.rb index b0905b8..d37b6ce 100644 --- a/lib/leave-one-out-validation.rb +++ b/lib/leave-one-out-validation.rb @@ -16,18 +16,11 @@ module OpenTox loo = klass.new :model_id => model.id predictions = model.predict model.training_dataset.substances predictions.each{|cid,p| p.delete(:neighbors)} - nr_unpredicted = 0 predictions.each do |cid,prediction| - if prediction[:value] - prediction[:measurements] = model.training_dataset.values(cid, prediction[:prediction_feature_id]) - else - nr_unpredicted += 1 - end + prediction[:measurements] = model.training_dataset.values(cid, prediction[:prediction_feature_id]) if prediction[:value] predictions.delete(cid) unless prediction[:value] and prediction[:measurements] end predictions.select!{|cid,p| p[:value] and p[:measurements]} - loo.nr_instances = predictions.size - loo.nr_unpredicted = nr_unpredicted loo.predictions = predictions loo.statistics $logger.debug "#{model.name}, LOO validation: #{Time.now-t} seconds" -- cgit v1.2.3 From 7e547fd4a296f497615a7805d565b378cb1bd7cd Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Wed, 14 Nov 2018 17:33:44 +0100 Subject: bad_request_error substituted with ArgumentError --- lib/leave-one-out-validation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/leave-one-out-validation.rb') diff --git a/lib/leave-one-out-validation.rb b/lib/leave-one-out-validation.rb index d37b6ce..7d73b89 100644 --- a/lib/leave-one-out-validation.rb +++ b/lib/leave-one-out-validation.rb @@ -9,7 +9,7 @@ module OpenTox # @param [OpenTox::Model::Lazar] # @return [OpenTox::Validation::LeaveOneOut] def self.create model - bad_request_error "Cannot create leave one out validation for models with supervised feature selection. Please use crossvalidation instead." if model.algorithms[:feature_selection] + raise ArgumentError, "Cannot create leave one out validation for models with supervised feature selection. Please use crossvalidation instead." if model.algorithms[:feature_selection] $logger.debug "#{model.name}: LOO validation started" t = Time.now model.training_dataset.features.collect{|f| f.class}.include?(NominalBioActivity) ? klass = ClassificationLeaveOneOut : klass = RegressionLeaveOneOut -- cgit v1.2.3