rewrite: adjust to new prediction dataset format from lazar, read confidence from ambit algorithms

8 files changed, 163 insertions, 79 deletions

diff --git a/lib/feature_util.rb b/lib/feature_util.rb
new file mode 100644
index 0000000..405d991
--- /dev/null
+++ b/lib/feature_util.rb
@@ -0,0 +1,66 @@
+
+module Lib
+  class FeatureUtil
+    
+    # this derieves the predicted_variable and predicted_confidence in prediction dataset
+    # predicted_variable: the feature-uri of model predictions
+    # predicted_confidence: the feature-uri of the model prediction confidence
+    # according to API info should be available in the API
+    # problem: IST has no feature-service -> predicted_variable depends on prediction dataset 
+    #
+    # PENDING: ambit and ist declare prediction features differently -> unify
+    #
+    def self.predicted_variables( model, prediction_dataset_uri, subjectid=nil )
+      
+      predicted_variable = nil
+      predicted_confidence = nil
+      
+      if model.metadata[OT.predictedVariables]
+        predictedVariables = model.metadata[OT.predictedVariables]
+        if predictedVariables.is_a?(Array)
+          if (predictedVariables.size==1)
+            predicted_variable = predictedVariables[0]
+          elsif (predictedVariables.size==2)
+            # PENDING identify confidence
+            conf_index = -1
+            predictedVariables.size.times do |i|
+              conf_index = i if OpenTox::Feature.find(predictedVariables[i]).metadata[DC.title]=~/(?i)confidence/
+            end
+            raise "size=2, no confidence "+predictedVariables.inspect+" "+model.uri.to_s if conf_index==-1
+            predicted_variable = predictedVariables[1-conf_index]
+            predicted_confidence = predictedVariables[conf_index]
+          else
+            raise "size>2 "+predictedVariables.inspect+" "+model.uri.to_s  
+          end
+        else
+          raise "predictedVariables is no array"
+        end        
+      end
+      
+      unless predicted_variable
+        d = OpenTox::Dataset.new prediction_dataset_uri
+        d.load_features(subjectid)
+        d.features.keys.each do |f|
+          if d.features[f][OT.hasSource]==model.uri
+            puts "source matching"
+            # PENDING identify confidence
+            if f =~ /(?i)confidence/
+              puts "conf matiching"
+              raise "duplicate confidence feature, what to choose?" if predicted_confidence!=nil
+              predicted_confidence = f
+            elsif d.features[f][RDF.type].include? OT.ModelPrediction
+              puts "type include prediction"
+              raise "duplicate predicted variable, what to choose?" if predicted_variable!=nil
+              predicted_variable = f
+            end
+          end
+          puts d.features[f][OT.hasSource]
+        end
+        raise "could not estimate predicted variable" unless predicted_variable
+      end
+      
+      {:predicted_variable => predicted_variable, :predicted_confidence => predicted_confidence}
+    end
+  end
+end
+  
diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb
index 2e3c238..df1efc3 100755
--- a/lib/ot_predictions.rb
+++ b/lib/ot_predictions.rb
@@ -16,16 +16,18 @@ module Lib
     end
   
     def initialize( feature_type, test_dataset_uris, test_target_dataset_uris, 
-      prediction_feature, prediction_dataset_uris, predicted_variables, subjectid=nil, task=nil)
+      prediction_feature, prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid=nil, task=nil)      
       
         test_dataset_uris = [test_dataset_uris] unless test_dataset_uris.is_a?(Array)
         test_target_dataset_uris = [test_target_dataset_uris] unless test_target_dataset_uris.is_a?(Array)
         prediction_dataset_uris = [prediction_dataset_uris] unless prediction_dataset_uris.is_a?(Array)
         predicted_variables = [predicted_variables] unless predicted_variables.is_a?(Array)
+        predicted_confidences = [predicted_confidences] unless predicted_confidences.is_a?(Array)
         LOGGER.debug "loading prediciton -- test-dataset:       "+test_dataset_uris.inspect
         LOGGER.debug "loading prediciton -- test-target-datset: "+test_target_dataset_uris.inspect
         LOGGER.debug "loading prediciton -- prediction-dataset: "+prediction_dataset_uris.inspect
         LOGGER.debug "loading prediciton -- predicted_variable: "+predicted_variables.inspect
+        LOGGER.debug "loading prediciton -- predicted_confidence: "+predicted_confidences.inspect
         LOGGER.debug "loading prediciton -- prediction_feature: "+prediction_feature.to_s
         raise "prediction_feature missing" unless prediction_feature
         
@@ -46,6 +48,7 @@ module Lib
           test_target_dataset_uri = test_target_dataset_uris[i]
           prediction_dataset_uri = prediction_dataset_uris[i]
           predicted_variable = predicted_variables[i]
+          predicted_confidence = predicted_confidences[i]
           
           predicted_variable=prediction_feature if predicted_variable==nil
         
@@ -72,7 +75,7 @@ module Lib
                   "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+
                   "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil
           end
-            
+          
           compounds = test_dataset.compounds
           LOGGER.debug "test dataset size: "+compounds.size.to_s
           raise "test dataset is empty "+test_dataset_uri.to_s unless compounds.size>0
@@ -92,35 +95,24 @@ module Lib
           compounds.each do |c|
             case feature_type
             when "classification"
-              actual_values << classification_value(test_target_dataset, c, prediction_feature, accept_values)
+              actual_values << classification_val(test_target_dataset, c, prediction_feature, accept_values)
             when "regression"
-              actual_values << regression_value(test_target_dataset, c, prediction_feature)
+              actual_values << regression_val(test_target_dataset, c, prediction_feature)
             end
           end
           task.progress( task_status += task_step ) if task # loaded actual values
         
           prediction_dataset = Lib::DatasetCache.find prediction_dataset_uri,subjectid
           raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset
-        
-          # TODO: remove LAZAR_PREDICTION_DATASET_HACK
-          no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil
-          if no_prediction_feature
-            one_entry_per_compound = true
-            compounds.each do |c|
-              if prediction_dataset.data_entries[c] and prediction_dataset.data_entries[c].size != 1
-                one_entry_per_compound = false
-                break
-              end
-            end
-            msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+
-              prediction_dataset.features.keys.inspect
-            if one_entry_per_compound
-              LOGGER.warn msg
-            else
-              raise msg
-            end
-          end
-  
+          raise "predicted_variable not found in prediction_dataset\n"+
+                  "predicted_variable '"+predicted_variable.to_s+"'\n"+
+                  "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+
+                  "available features are: "+prediction_dataset.features.inspect if prediction_dataset.features.keys.index(predicted_variable)==nil
+          raise "predicted_confidence not found in prediction_dataset\n"+
+                  "predicted_confidence '"+predicted_confidence.to_s+"'\n"+
+                  "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+
+                  "available features are: "+prediction_dataset.features.inspect if predicted_confidence and prediction_dataset.features.keys.index(predicted_confidence)==nil
+
           raise "more predicted than test compounds, #test: "+compounds.size.to_s+" < #prediction: "+
             prediction_dataset.compounds.size.to_s+", test-dataset: "+test_dataset_uri.to_s+", prediction-dataset: "+
              prediction_dataset_uri if compounds.size < prediction_dataset.compounds.size
@@ -141,25 +133,18 @@ module Lib
             else
               case feature_type
               when "classification"
-                # TODO: remove LAZAR_PREDICTION_DATASET_HACK
-                predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, accept_values)
+                predicted_values << classification_val(prediction_dataset, c, predicted_variable, accept_values)
               when "regression"
-                predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable)
+                predicted_values << regression_val(prediction_dataset, c, predicted_variable)
               end
-              # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable)
-              conf = predicted_values[count]!=nil ? 1 : nil 
-              begin
-                feature = prediction_dataset.data_entries[c].keys[0]
-                feature_data = prediction_dataset.features[feature]
-                conf = feature_data[OT.confidence].to_f if feature_data[OT.confidence]!=nil 
-              rescue
-                LOGGER.warn "could not get confidence"
+              if predicted_confidence
+                confidence_values << confidence_val(prediction_dataset, c, predicted_confidence)
+              else
+                confidence_values << nil
               end
-              confidence_values << conf
             end
             count += 1
           end
-          
           @compounds += compounds
           all_predicted_values += predicted_values
           all_actual_values += actual_values
@@ -174,7 +159,7 @@ module Lib
     end
     
     private
-    def regression_value(dataset, compound, feature)
+    def regression_val(dataset, compound, feature)
       v = value(dataset, compound, feature)
       begin
         v = v.to_f unless v==nil or v.is_a?(Numeric)
@@ -185,7 +170,18 @@ module Lib
       end
     end
     
-    def classification_value(dataset, compound, feature, accept_values)
+    def confidence_val(dataset, compound, confidence)
+      v = value(dataset, compound, confidence)
+      begin
+        v = v.to_f unless v==nil or v.is_a?(Numeric)
+        v
+      rescue
+        LOGGER.warn "no numeric value for confidence '"+v.to_s+"'"
+        nil
+      end
+    end
+    
+    def classification_val(dataset, compound, feature, accept_values)
       v = value(dataset, compound, feature)
       i = accept_values.index(v.to_s)
       raise "illegal class_value of prediction (value is '"+v.to_s+"'), accept values are "+
@@ -239,6 +235,10 @@ module Lib
     
     def self.to_array( predictions, add_pic=false, format=false )
   
+      confidence_available = false
+      predictions.each do |p|
+        confidence_available |= p.confidence_values_available?
+      end
       res = []
       conf_column = nil
       predictions.each do |p|
@@ -269,9 +269,9 @@ module Lib
               a << nil
             end
           end
-          if p.confidence_values_available?
+          if confidence_available
             conf_column = a.size if conf_column==nil
-            a << p.confidence_value(i) #(format ? p.confidence_value(i).to_nice_s : p.confidence_value(i))
+            a << p.confidence_value(i)
           end
           a << p.identifier(i)
           res << a
@@ -279,6 +279,7 @@ module Lib
       end
       
       if conf_column!=nil
+        LOGGER.debug "sort via confidence: "+res.collect{|n| n[conf_column]}.inspect
         res = res.sort_by{ |n| n[conf_column] || 0 }.reverse
         if format
           res.each do |a|
diff --git a/lib/predictions.rb b/lib/predictions.rb
index 4a15e1d..a449776 100755
--- a/lib/predictions.rb
+++ b/lib/predictions.rb
@@ -45,16 +45,6 @@ module Lib
       raise "illegal num confidence values "+num_info if  @confidence_values.size != @predicted_values.size
       
       @confidence_values.each{ |c| raise "illegal confidence value: '"+c.to_s+"'" unless c==nil or (c.is_a?(Numeric) and c>=0 and c<=1) }
-      ## check if there is more than one different conf value
-      ## DEPRECATED? not sure anymore what this was about, 
-      ##             I am pretty sure this was for r-plot of roc curves
-      ##             roc curvers are now plotted manually
-      #conf_val_tmp = {}
-      #@confidence_values.each{ |c| conf_val_tmp[c] = nil }
-      #if conf_val_tmp.keys.size<2
-      #  LOGGER.warn("prediction w/o confidence values");
-      #  @confidence_values=nil
-      #end
       
       case @feature_type
       when "classification"
@@ -75,11 +65,13 @@ module Lib
       
       init_stats()
       (0..@predicted_values.size-1).each do |i|
-        update_stats( @predicted_values[i], @actual_values[i], (@confidence_values!=nil)?@confidence_values[i]:nil )
+        update_stats( @predicted_values[i], @actual_values[i], @confidence_values[i] )
       end
     end
     
     def init_stats
+      @conf_provided = false
+      
       @num_no_actual_value = 0
       @num_with_actual_value = 0 
       
@@ -134,6 +126,8 @@ module Lib
         else
           @num_predicted += 1
           
+          @conf_provided |= confidence_value!=nil
+          
           case @feature_type
           when "classification"
             @confusion_matrix[actual_value][predicted_value] += 1
@@ -186,6 +180,7 @@ module Lib
     end
     
     def weighted_accuracy
+      return 0 unless confidence_values_available?      
       raise "no classification" unless @feature_type=="classification"
       total = 0
       correct = 0
@@ -255,7 +250,7 @@ module Lib
     def area_under_roc(class_index=nil)
       return prediction_feature_value_map( lambda{ |i| area_under_roc(i) } ) if 
         class_index==nil
-      return 0.0 if @confidence_values==nil
+      return 0 unless confidence_values_available?
       
       LOGGER.warn("TODO: implement approx computiation of AUC,"+
         "so far Wilcoxon-Man-Whitney is used (exponential)") if 
@@ -485,7 +480,9 @@ module Lib
       
       # see http://en.wikipedia.org/wiki/Coefficient_of_determination#Definitions
       # see http://web.maths.unsw.edu.au/~adelle/Garvan/Assays/GoodnessOfFit.html
-      r_2 = 1 - residual_sum_of_squares / total_sum_of_squares
+      ss_tot = total_sum_of_squares
+      return 0 if ss_tot==0
+      r_2 = 1 - residual_sum_of_squares / ss_tot
       ( r_2.infinite? || r_2.nan? ) ? 0 : r_2
     end
     
@@ -523,7 +520,7 @@ module Lib
     def get_prediction_values(class_value)
       
       #puts "get_roc_values for class_value: "+class_value.to_s
-      raise "no confidence values" if @confidence_values==nil
+      raise "no confidence values" unless confidence_values_available?
       #raise "no class-value specified" if class_value==nil
       
       class_index = @accept_values.index(class_value) if class_value!=nil
@@ -594,7 +591,7 @@ module Lib
     end
     
     def confidence_values_available?
-      return @confidence_values!=nil
+      @conf_provided
     end
     
     ###################################################################################################################
diff --git a/report/report_content.rb b/report/report_content.rb
index d60f700..755147d 100755
--- a/report/report_content.rb
+++ b/report/report_content.rb
@@ -287,12 +287,9 @@ class Reports::ReportContent
     
     section_bar = @xml_report.add_section(@current_section, section_title)
     @xml_report.add_paragraph(section_bar, section_text) if section_text
-    
-    plot_file_name = "bar_plot"+@tmp_file_count.to_s+".png"
-    @tmp_file_count += 1
-    plot_file_path = add_tmp_file(plot_file_name)
-    Reports::PlotFactory.create_bar_plot(plot_file_path, validation_set, title_attribute, value_attributes )
-    @xml_report.add_imagefigure(section_bar, image_title, plot_file_name, "PNG", 100, image_caption)
+    plot_bar = add_tmp_file("bar_plot", "svg")
+    Reports::PlotFactory.create_bar_plot(plot_bar[:path], validation_set, title_attribute, value_attributes )
+    @xml_report.add_imagefigure(section_bar, image_title, plot_bar[:name], "PNG", 100, image_caption)
   end  
   
   private
diff --git a/report/validation_access.rb b/report/validation_access.rb
index ffb7461..d421a0b 100755
--- a/report/validation_access.rb
+++ b/report/validation_access.rb
@@ -100,15 +100,17 @@ class Reports::ValidationDB
   def get_predictions(validation, subjectid=nil, task=nil)
     Lib::OTPredictions.new( validation.feature_type, validation.test_dataset_uri, 
       validation.test_target_dataset_uri, validation.prediction_feature, validation.prediction_dataset_uri, 
-      validation.predicted_variable, subjectid, task)
+      validation.predicted_variable, validation.predicted_confidence, subjectid, task)
   end
   
   def get_accept_values( validation, subjectid=nil )
     # PENDING So far, one has to load the whole dataset to get the accept_value from ambit
-    d = Lib::DatasetCache.find( validation.test_target_dataset_uri, subjectid )
-    raise "cannot get test target dataset for accept values, dataset: "+validation.test_target_dataset_uri.to_s unless d
+    test_target_dataset = validation.test_target_dataset_uri
+    test_target_dataset = validation.test_dataset_uri unless test_target_dataset
+    d = Lib::DatasetCache.find( test_target_dataset, subjectid )
+    raise "cannot get test target dataset for accept values, dataset: "+test_target_dataset.to_s unless d
     accept_values = d.features[validation.prediction_feature][OT.acceptValue]
-    raise "cannot get accept values from dataset "+validation.test_target_dataset_uri.to_s+" for feature "+
+    raise "cannot get accept values from dataset "+test_target_dataset.to_s+" for feature "+
       validation.prediction_feature+":\n"+d.features[validation.prediction_feature].to_yaml unless accept_values!=nil
     accept_values
   end
@@ -122,10 +124,16 @@ class Reports::ValidationDB
     raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation)
     model = OpenTox::Model::Generic.find(validation.model_uri, subjectid)
     raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model
-    model.metadata[OT.predictedVariables]
-    #get_model(validation).predictedVariables
+    Lib::FeatureUtil.predicted_variables(model, validation.prediction_dataset_uri, subjectid)[:predicted_variable]
   end
   
+  def predicted_confidence(validation, subjectid=nil)
+    raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation)
+    model = OpenTox::Model::Generic.find(validation.model_uri, subjectid)
+    raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model
+    Lib::FeatureUtil.predicted_variables(model, validation.prediction_dataset_uri, subjectid)[:predicted_confidence]
+  end  
+  
 #  private
 #  def get_model(validation)
 #    raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation)
diff --git a/report/validation_data.rb b/report/validation_data.rb
index 0c15dd2..2479977 100755
--- a/report/validation_data.rb
+++ b/report/validation_data.rb
@@ -139,6 +139,11 @@ module Reports
       @predicted_variable = Reports.validation_access.predicted_variable(self, @subjectid) 
     end
     
+    def predicted_confidence
+      return @predicted_confidence if @predicted_confidence!=nil
+      @predicted_confidence = Reports.validation_access.predicted_confidence(self, @subjectid) 
+    end    
+    
     # loads all crossvalidation attributes, of the corresponding cv into this object 
     def load_cv_attributes
       raise "crossvalidation-id not set" unless @crossvalidation_id
diff --git a/validation/validation_application.rb b/validation/validation_application.rb
index d448c62..f64f74e 100755
--- a/validation/validation_application.rb
+++ b/validation/validation_application.rb
@@ -4,6 +4,7 @@
 end
 
 require 'lib/dataset_cache.rb'
+require 'lib/feature_util.rb'
 require 'validation/validation_service.rb'
 
 get '/crossvalidation/?' do
@@ -443,21 +444,22 @@ post '/validate_datasets' do
     params[:validation_type] = "validate_datasets" 
     
     if params[:model_uri]
+      raise OpenTox::BadRequestError.new "please specify 'model_uri' or set either 'classification' or 'regression' flag" if params[:classification] or params[:regression]
       v = Validation::Validation.create params
       v.subjectid = @subjectid
       v.compute_validation_stats_with_model(nil,false,task)
     else
       raise OpenTox::BadRequestError.new "please specify 'model_uri' or 'prediction_feature'" unless params[:prediction_feature]
-      raise OpenTox::BadRequestError.new "please specify 'model_uri' or 'predicted_feature'" unless params[:predicted_feature]
+      raise OpenTox::BadRequestError.new "please specify 'model_uri' or 'predicted_variable'" unless params[:predicted_variable]
       raise OpenTox::BadRequestError.new "please specify 'model_uri' or set either 'classification' or 'regression' flag" unless 
             params[:classification] or params[:regression]
-      
-      predicted_feature = params.delete("predicted_feature")
+      predicted_variable = params.delete("predicted_variable")
+      predicted_confidence = params.delete("predicted_confidence")
       feature_type = "classification" if params.delete("classification")!=nil
       feature_type = "regression" if params.delete("regression")!=nil
       v = Validation::Validation.create params  
       v.subjectid = @subjectid
-      v.compute_validation_stats(feature_type,predicted_feature,nil,nil,false,task)
+      v.compute_validation_stats(feature_type,predicted_variable,predicted_confidence,nil,nil,false,task)
     end
     v.validation_uri
   end
diff --git a/validation/validation_service.rb b/validation/validation_service.rb
index 61f3a6e..73c15df 100755
--- a/validation/validation_service.rb
+++ b/validation/validation_service.rb
@@ -46,9 +46,15 @@ module Validation
         test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri}
         prediction_feature = vals.first.prediction_feature
         prediction_dataset_uris = vals.collect{|v| v.prediction_dataset_uri}
-        predicted_variables = models.collect{|m| m.metadata[OT.predictedVariables]}
+        predicted_variables = []
+        predicted_confidences = []
+        models.size.times do |i|
+          predicted = Lib::FeatureUtil.predicted_variables(models[i], prediction_dataset_uris[i], subjectid)
+          predicted_variables << predicted[:predicted_variable]
+          predicted_confidences << predicted[:predicted_confidence]
+        end
         prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, 
-          prediction_dataset_uris, predicted_variables, subjectid )
+          prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid )
           
         v = Validation.new
         case feature_type
@@ -218,13 +224,15 @@ module Validation
       dependentVariables = model.metadata[OT.dependentVariables]
       prediction_feature = self.prediction_feature ? nil : dependentVariables
       algorithm_uri = self.algorithm_uri ? nil : model.metadata[OT.algorithm]
-      predictedVariables = model.metadata[OT.predictedVariables]
-      compute_validation_stats( model.feature_type(self.subjectid), predictedVariables, 
+      predicted_variables = Lib::FeatureUtil.predicted_variables(model, prediction_dataset_uri, subjectid)
+      predicted_variable = predicted_variables[:predicted_variable]
+      predicted_confidence = predicted_variables[:predicted_confidence]
+      compute_validation_stats( model.feature_type(self.subjectid), predicted_variable, predicted_confidence, 
         prediction_feature, algorithm_uri, dry_run, task )
     end
       
-    def compute_validation_stats( feature_type, predicted_feature, prediction_feature=nil, 
-        algorithm_uri=nil, dry_run=false, task=nil )
+    def compute_validation_stats( feature_type, predicted_variable, predicted_confidence, prediction_feature, 
+        algorithm_uri, dry_run, task )        
       
 #      self.attributes = { :prediction_feature => prediction_feature } if self.prediction_feature==nil && prediction_feature
 #      self.attributes = { :algorithm_uri => algorithm_uri } if self.algorithm_uri==nil && algorithm_uri
@@ -237,7 +245,7 @@ module Validation
       LOGGER.debug "computing prediction stats"
       prediction = Lib::OTPredictions.new( feature_type, 
         self.test_dataset_uri, self.test_target_dataset_uri, self.prediction_feature, 
-        self.prediction_dataset_uri, predicted_feature, self.subjectid, OpenTox::SubTask.create(task, 0, 80) )
+        self.prediction_dataset_uri, predicted_variable, predicted_confidence, self.subjectid, OpenTox::SubTask.create(task, 0, 80) )
       #reading datasets and computing the main stats is 80% the work 
       
       unless dry_run