1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
|
module OpenTox
module Validation
# Leave one out validation
class LeaveOneOut < Validation
# Create a leave one out validation
# @param [OpenTox::Model::Lazar]
# @return [OpenTox::Validation::LeaveOneOut]
def self.create model
raise ArgumentError, "Cannot create leave one out validation for models with supervised feature selection. Please use crossvalidation instead." if model.algorithms[:feature_selection]
$logger.debug "#{model.name}: LOO validation started"
t = Time.now
model.training_dataset.features.collect{|f| f.class}.include?(NominalBioActivity) ? klass = ClassificationLeaveOneOut : klass = RegressionLeaveOneOut
loo = klass.new :model_id => model.id
predictions = model.predict model.training_dataset.substances
predictions.each{|cid,p| p.delete(:neighbors)}
predictions.each do |cid,prediction|
prediction[:measurements] = model.training_dataset.values(cid, prediction[:prediction_feature_id]) if prediction[:value]
predictions.delete(cid) unless prediction[:value] and prediction[:measurements]
end
predictions.select!{|cid,p| p[:value] and p[:measurements]}
loo.predictions = predictions
loo.statistics
$logger.debug "#{model.name}, LOO validation: #{Time.now-t} seconds"
loo
end
end
# Leave one out validation for classification models
class ClassificationLeaveOneOut < LeaveOneOut
include ClassificationStatistics
field :accept_values, type: Array
field :confusion_matrix, type: Hash
field :weighted_confusion_matrix, type: Hash
field :accuracy, type: Hash
field :weighted_accuracy, type: Hash
field :true_rate, type: Hash
field :predictivity, type: Hash
field :nr_predictions, type: Hash
field :probability_plot_id, type: BSON::ObjectId
end
# Leave one out validation for regression models
class RegressionLeaveOneOut < LeaveOneOut
include RegressionStatistics
field :rmse, type: Hash
field :mae, type: Hash
field :r_squared, type: Hash
field :within_prediction_interval, type: Hash
field :out_of_prediction_interval, type: Hash
field :nr_predictions, type: Hash
field :warnings, type: Array
field :correlation_plot_id, type: BSON::ObjectId
end
end
end
|
