From b515a0cfedb887a2af753db6e4a08ae1af430cad Mon Sep 17 00:00:00 2001
From: Christoph Helma <helma@in-silico.ch>
Date: Tue, 31 May 2016 18:08:08 +0200
Subject: cleanup of validation modules/classes

---
 lib/train-test-validation.rb | 58 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 lib/train-test-validation.rb

(limited to 'lib/train-test-validation.rb')

diff --git a/lib/train-test-validation.rb b/lib/train-test-validation.rb
new file mode 100644
index 0000000..286614a
--- /dev/null
+++ b/lib/train-test-validation.rb
@@ -0,0 +1,58 @@
+module OpenTox
+
+  module Validation
+
+    class TrainTest < Validation
+
+      field :training_dataset_id, type: BSON::ObjectId
+      field :test_dataset_id, type: BSON::ObjectId
+
+      def self.create model, training_set, test_set
+        
+        atts = model.attributes.dup # do not modify attributes of the original model
+        atts["_id"] = BSON::ObjectId.new
+        atts[:training_dataset_id] = training_set.id
+        validation_model = model.class.create model.prediction_feature, training_set, atts
+        validation_model.save
+        predictions = validation_model.predict test_set.substances
+        nr_unpredicted = 0
+        predictions.each do |cid,prediction|
+          if prediction[:value]
+            prediction[:measurements] = test_set.values(cid, prediction[:prediction_feature_id])
+          else
+            nr_unpredicted += 1
+          end
+        end
+        predictions.select!{|cid,p| p[:value] and p[:measurements]}
+        validation = self.new(
+          :model_id => validation_model.id,
+          :test_dataset_id => test_set.id,
+          :nr_instances => test_set.substances.size,
+          :nr_unpredicted => nr_unpredicted,
+          :predictions => predictions
+        )
+        validation.save
+        validation
+      end
+
+      def test_dataset
+        Dataset.find test_dataset_id
+      end
+
+      def training_dataset
+        Dataset.find training_dataset_id
+      end
+
+    end
+
+    class ClassificationTrainTest < TrainTest
+      include ClassificationStatistics
+    end
+
+    class RegressionTrainTest < TrainTest
+      include RegressionStatistics
+    end
+
+  end
+
+end
-- 
cgit v1.2.3


From 4348eec89033e6677c9f628646fc67bd03c73fe6 Mon Sep 17 00:00:00 2001
From: Christoph Helma <helma@in-silico.ch>
Date: Thu, 6 Oct 2016 19:14:10 +0200
Subject: nano caret regression fixed

---
 lib/train-test-validation.rb | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'lib/train-test-validation.rb')

diff --git a/lib/train-test-validation.rb b/lib/train-test-validation.rb
index 286614a..e3f5905 100644
--- a/lib/train-test-validation.rb
+++ b/lib/train-test-validation.rb
@@ -9,10 +9,7 @@ module OpenTox
 
       def self.create model, training_set, test_set
         
-        atts = model.attributes.dup # do not modify attributes of the original model
-        atts["_id"] = BSON::ObjectId.new
-        atts[:training_dataset_id] = training_set.id
-        validation_model = model.class.create model.prediction_feature, training_set, atts
+        validation_model = model.class.create prediction_feature: model.prediction_feature, training_dataset: training_set, algorithms: model.algorithms
         validation_model.save
         predictions = validation_model.predict test_set.substances
         nr_unpredicted = 0
-- 
cgit v1.2.3


From 8519274487166d75b3b9ae28e61f7a7be9f7e83c Mon Sep 17 00:00:00 2001
From: Christoph Helma <helma@in-silico.ch>
Date: Thu, 27 Oct 2016 11:58:07 +0200
Subject: probability plot for classification validations

---
 lib/train-test-validation.rb | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

(limited to 'lib/train-test-validation.rb')

diff --git a/lib/train-test-validation.rb b/lib/train-test-validation.rb
index e3f5905..71abad2 100644
--- a/lib/train-test-validation.rb
+++ b/lib/train-test-validation.rb
@@ -44,10 +44,24 @@ module OpenTox
 
     class ClassificationTrainTest < TrainTest
       include ClassificationStatistics
+      field :accept_values, type: Array
+      field :confusion_matrix, type: Array
+      field :weighted_confusion_matrix, type: Array
+      field :accuracy, type: Float
+      field :weighted_accuracy, type: Float
+      field :true_rate, type: Hash
+      field :predictivity, type: Hash
+      field :probability_plot_id, type: BSON::ObjectId
     end
 
     class RegressionTrainTest < TrainTest
       include RegressionStatistics
+      field :rmse, type: Float, default:0
+      field :mae, type: Float, default:0
+      field :r_squared, type: Float
+      field :within_prediction_interval, type: Integer, default:0
+      field :out_of_prediction_interval, type: Integer, default:0
+      field :correlation_plot_id, type: BSON::ObjectId
     end
 
   end
-- 
cgit v1.2.3