crossvalidation confusion matrix multinominal

4 files changed, 60 insertions, 41 deletions

diff --git a/application.rb b/application.rb
index 54aeb3d..2e058bb 100644
--- a/application.rb
+++ b/application.rb
@@ -291,7 +291,7 @@ post '/models' do # create a new model
     if CONFIG[:services]["opentox-validation"]
       @model.update :status => "Validating model"
       begin
-        validation = OpenTox::Crossvalidation.create( {
+        crossvalidation = OpenTox::Crossvalidation.create( {
             :algorithm_uri => lazar.metadata[OT.algorithm],
             :dataset_uri => lazar.parameter("dataset_uri"),
             :subjectid => subjectid,
@@ -299,25 +299,37 @@ post '/models' do # create a new model
             :algorithm_params => "feature_generation_uri=#{lazar.parameter("feature_generation_uri")}" },
             nil, OpenTox::SubTask.new(task,25,80))
 
-        @model.update(:validation_uri => validation.uri)
+        @model.update(:validation_uri => crossvalidation.uri)
         LOGGER.debug "Validation URI: #{@model.validation_uri}"
 
         # create summary
-        validation.summary(subjectid).each do |k,v|
-          begin
-            eval "@model.update :#{k.to_s} => v" if v
-          rescue
-            eval "@model.update :#{k.to_s} => 0"
+        validation = crossvalidation.statistics(subjectid)
+        @model.update(:nr_predictions => validation.metadata[OT.numInstances].to_i - validation.metadata[OT.numUnpredicted].to_i)
+        if validation.metadata[OT.classificationStatistics]
+          @model.update(:correct_predictions => validation.metadata[OT.classificationStatistics][OT.percentCorrect].to_f)
+          @model.update(:confusion_matrix => validation.confusion_matrix.to_yaml)
+          @model.update(:weighted_area_under_roc => validation.metadata[OT.classificationStatistics][OT.weightedAreaUnderRoc].to_f)
+          validation.metadata[OT.classificationStatistics][OT.classValueStatistics].each do |m|
+            if m[OT.classValue] =~ TRUE_REGEXP
+              #HACK: estimate true feature value correctly 
+              @model.update(:sensitivity => m[OT.truePositiveRate])
+              @model.update(:specificity => m[OT.trueNegativeRate])
+              break
+            end
           end
+        else
+          @model.update(:r_square => validation.metadata[OT.regressionStatistics][OT.rSquare].to_f)
+          @model.update(:root_mean_squared_error => validation.metadata[OT.regressionStatistics][OT.rootMeanSquaredError].to_f)
+          @model.update(:mean_absolute_error => validation.metadata[OT.regressionStatistics][OT.meanAbsoluteError].to_f)
         end
       rescue => e
-        @model.update :warnings => @model.warnings.to_s+"\nModel validation failed with #{e.message}."
+        @model.update :warnings => @model.warnings.to_s+"\nModel crossvalidation failed with #{e.message}."
         error "Model validation failed",e
       end
     
       begin
         @model.update :status => "Creating validation report"
-        validation_report_uri = validation.find_or_create_report(subjectid, OpenTox::SubTask.new(task,80,90)) #unless @model.dirty?
+        validation_report_uri = crossvalidation.find_or_create_report(subjectid, OpenTox::SubTask.new(task,80,90)) #unless @model.dirty?
         @model.update :validation_report_uri => validation_report_uri, :status => "Creating QMRF report"
         qmrf_report = OpenTox::Crossvalidation::QMRFReport.create(@model.uri, subjectid, OpenTox::SubTask.new(task,90,99))
         @model.update(:validation_qmrf_uri => qmrf_report.uri, :status => "Completed")
diff --git a/helper.rb b/helper.rb
index 0acb90e..408f8bc 100644
--- a/helper.rb
+++ b/helper.rb
@@ -53,7 +53,7 @@ helpers do
 
   def sort(descriptors)
     features = {:activating => [], :deactivating => []}
-    descriptors.each do |d| 
+    descriptors.each do |d|
       if d[OT.effect] =~ TRUE_REGEXP
         features[:activating] << {:smarts => d[OT.smarts],:p_value => d[OT.pValue]} 
       elsif d[OT.effect] =~ FALSE_REGEXP
diff --git a/model.rb b/model.rb
index 5b1f4f5..e433893 100644
--- a/model.rb
+++ b/model.rb
@@ -27,14 +27,10 @@ class ToxCreateModel < Ohm::Model
 
 	#attributey :validation_qmrf_task_uri
 	attribute :validation_qmrf_uri
-
+  attribute :confusion_matrix
 	attribute :nr_compounds
-	attribute :nr_predictions
-	attribute :true_positives
-	attribute :false_positives
-	attribute :true_negatives
-	attribute :false_negatives
-	attribute :correct_predictions
+  attribute :nr_predictions
+  attribute :correct_predictions
 	attribute :weighted_area_under_roc
 	attribute :sensitivity
 	attribute :specificity
diff --git a/views/classification_validation.haml b/views/classification_validation.haml
index 5649d7f..f25a321 100644
--- a/views/classification_validation.haml
+++ b/views/classification_validation.haml
@@ -1,33 +1,44 @@
 %dt Correct predictions:
 %dd
-	= sprintf("%.2f", model.correct_predictions.to_f) if model.correct_predictions
-	= '%'
+  = sprintf("%.2f", model.correct_predictions.to_f) if model.correct_predictions
+  = '%'
 %dt
-	%a{:href => "http://en.wikipedia.org/wiki/Receiver_operating_characteristic", :rel => "external"} Weighted area under ROC:
+  %a{:href => "http://en.wikipedia.org/wiki/Receiver_operating_characteristic", :rel => "external"} Weighted area under ROC:
 %dd 
-	= sprintf("%.3f", model.weighted_area_under_roc.to_f) if model.weighted_area_under_roc
+  = sprintf("%.3f", model.weighted_area_under_roc.to_f) if model.weighted_area_under_roc
 %dt
-	%a{:href => "http://en.wikipedia.org/wiki/Sensitivity_and_specificity", :rel => "external"} Specificity:
+  %a{:href => "http://en.wikipedia.org/wiki/Sensitivity_and_specificity", :rel => "external"} Specificity:
 %dd= sprintf("%.3f", model.specificity.to_f) if model.specificity
 %dt
-	%a{:href => "http://en.wikipedia.org/wiki/Sensitivity_and_specificity", :rel => "external"} Sensitivity:
+  %a{:href => "http://en.wikipedia.org/wiki/Sensitivity_and_specificity", :rel => "external"} Sensitivity:
 %dd= sprintf("%.3f", model.sensitivity.to_f) if model.sensitivity
 %dt
-	%a{:href => "http://en.wikipedia.org/wiki/Confusion_matrix", :rel => "external"} Confusion Matrix:
-%dd
-	%table
-		%tr
-			%td{:colspan => 2, :rowspan => 2}
-			%th{:colspan => 2} Measured
-		%tr
-			%th{:bgcolor => "#CCD2DC"} active
-			%th{:bgcolor => "#CCD2DC"} inactive
-		%tr
-			%th{:rowspan => 2} Predicted
-			%th{:bgcolor => "#CCD2DC"} active
-			%td= model.true_positives if model.true_positives
-			%td= model.false_positives if model.false_positives
-		%tr
-			%th{:bgcolor => "#CCD2DC"} inactive
-			%td= model.false_negatives if model.false_negatives
-			%td= model.true_negatives if model.true_negatives
+  %a{:href => "http://en.wikipedia.org/wiki/Confusion_matrix", :rel => "external"} Confusion Matrix:
+- if model.confusion_matrix
+  - cm = YAML.load(model.confusion_matrix)
+  %dd
+    %table
+      %tr
+        %td{:colspan => 2, :rowspan => 2}
+        %th{:colspan => (cm.size - 1)} Measured
+      %tr
+        - (1..cm[0].size-1).each do |i|
+          %th{:bgcolor => "#CCD2DC"}
+            = cm[0][i]
+      %tr
+        %th{:rowspan => (cm.size - 1)} Predicted
+        - (0..cm[1].size-1).each do |i|
+          - if i == 0
+            %th{:bgcolor => "#CCD2DC"}
+              = cm[1][i]
+          - else
+            %td= cm[1][i]
+      - if cm.size > 2
+        - (2..cm.size-1).each do |i|
+          %tr
+            - (0..cm[i].size-1).each do |j|
+              - if j == 0
+                %th{:bgcolor => "#CCD2DC"}
+                  = cm[i][j]
+              - else
+                %td= cm[i][j]