summaryrefslogtreecommitdiff
path: root/lib/leave-one-out-validation.rb
blob: c33c92bedda1be2fef314f4394510147586f40ae (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
module OpenTox

  module Validation

    # Leave one out validation
    class LeaveOneOut < Validation

      # Create a leave one out validation
      # @param [OpenTox::Model::Lazar]
      # @return [OpenTox::Validation::LeaveOneOut]
      def self.create model
        bad_request_error "Cannot create leave one out validation for models with supervised feature selection. Please use crossvalidation instead." if model.algorithms[:feature_selection]
        $logger.debug "#{model.name}: LOO validation started"
        t = Time.now
        model.training_dataset.features.first.nominal? ? klass = ClassificationLeaveOneOut : klass = RegressionLeaveOneOut
        loo = klass.new :model_id => model.id
        predictions = model.predict model.training_dataset.substances
        predictions.each{|cid,p| p.delete(:neighbors)}
        nr_unpredicted = 0
        predictions.each do |cid,prediction|
          if prediction[:value]
            prediction[:measurements] = model.training_dataset.values(cid, prediction[:prediction_feature_id])
          else
            nr_unpredicted += 1
          end
          predictions.delete(cid) unless prediction[:value] and prediction[:measurements]
        end
        predictions.select!{|cid,p| p[:value] and p[:measurements]}
        loo.nr_instances = predictions.size
        loo.nr_unpredicted = nr_unpredicted
        loo.predictions = predictions
        loo.statistics
        $logger.debug "#{model.name}, LOO validation:  #{Time.now-t} seconds"
        loo
      end

    end

    # Leave one out validation for classification models
    class ClassificationLeaveOneOut < LeaveOneOut
      include ClassificationStatistics
      field :accept_values, type: Array
      field :confusion_matrix, type: Array, default: []
      field :weighted_confusion_matrix, type: Array, default: []
      field :accuracy, type: Float
      field :weighted_accuracy, type: Float
      field :true_rate, type: Hash, default: {}
      field :predictivity, type: Hash, default: {}
      field :confidence_plot_id, type: BSON::ObjectId
    end
    
    # Leave one out validation for regression models
    class RegressionLeaveOneOut  < LeaveOneOut
      include RegressionStatistics
      field :rmse, type: Float, default: 0
      field :mae, type: Float, default: 0
      field :r_squared, type: Float
      field :within_prediction_interval, type: Integer, default:0
      field :out_of_prediction_interval, type: Integer, default:0
      field :correlation_plot_id, type: BSON::ObjectId
      field :warnings, type: Array
    end

  end

end