crossvalidation statistics fix: compute cv-statistics with cv-predictions instead of averaging cv-validation-statistics

10 files changed, 322 insertions, 210 deletions

diff --git a/lib/dataset_cache.rb b/lib/dataset_cache.rb
new file mode 100644
index 0000000..1af1d51
--- /dev/null
+++ b/lib/dataset_cache.rb
@@ -0,0 +1,23 @@
+
+module Lib
+  
+  module DatasetCache
+    
+    @@cache={}
+
+    # same as OpenTox::Dataset.find with caching function
+    # rational: datasets are reused in crossvalidation very often, cache to save computational effort
+    # PENDING: may cause memory issues, test with huge datasets 
+    def self.find(dataset_uri, subjectid=nil)
+      return nil if (dataset_uri==nil)
+      d = @@cache[dataset_uri.to_s+"_"+subjectid.to_s]
+      if d==nil
+        d = OpenTox::Dataset.find(dataset_uri, subjectid)
+        @@cache[dataset_uri.to_s+"_"+subjectid.to_s] = d
+      end
+      d
+    end
+    
+  end
+  
+end
\ No newline at end of file
diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb
index 22f9b20..644168f 100755
--- a/lib/ot_predictions.rb
+++ b/lib/ot_predictions.rb
@@ -15,127 +15,161 @@ module Lib
       return @compounds[instance_index]
     end
   
-    def initialize(feature_type, test_dataset_uri, test_target_dataset_uri, 
-      prediction_feature, prediction_dataset_uri, predicted_variable, subjectid=nil, task=nil)
+    def initialize( feature_type, test_dataset_uris, test_target_dataset_uris, 
+      prediction_feature, prediction_dataset_uris, predicted_variables, subjectid=nil, task=nil)
       
-        LOGGER.debug("loading prediciton via test-dataset:'"+test_dataset_uri.to_s+
-          "', test-target-datset:'"+test_target_dataset_uri.to_s+
-          "', prediction-dataset:'"+prediction_dataset_uri.to_s+
-          "', prediction_feature: '"+prediction_feature.to_s+"' "+
-          "', predicted_variable: '"+predicted_variable.to_s+"'")
-          
-        predicted_variable=prediction_feature if predicted_variable==nil
-        
-        test_dataset = OpenTox::Dataset.find test_dataset_uri,subjectid
-        raise "test dataset not found: '"+test_dataset_uri.to_s+"'" unless test_dataset
+        test_dataset_uris = [test_dataset_uris] unless test_dataset_uris.is_a?(Array)
+        test_target_dataset_uris = [test_target_dataset_uris] unless test_target_dataset_uris.is_a?(Array)
+        prediction_dataset_uris = [prediction_dataset_uris] unless prediction_dataset_uris.is_a?(Array)
+        predicted_variables = [predicted_variables] unless predicted_variables.is_a?(Array)
+        LOGGER.debug "loading prediciton -- test-dataset:       "+test_dataset_uris.inspect
+        LOGGER.debug "loading prediciton -- test-target-datset: "+test_target_dataset_uris.inspect
+        LOGGER.debug "loading prediciton -- prediction-dataset: "+prediction_dataset_uris.inspect
+        LOGGER.debug "loading prediciton -- predicted_variable: "+predicted_variables.inspect
+        LOGGER.debug "loading prediciton -- prediction_feature: "+prediction_feature.to_s
         raise "prediction_feature missing" unless prediction_feature
         
-        if test_target_dataset_uri == nil || test_target_dataset_uri.strip.size==0 || test_target_dataset_uri==test_dataset_uri
-          test_target_dataset_uri = test_dataset_uri
-          test_target_dataset = test_dataset
-          raise "prediction_feature not found in test_dataset, specify a test_target_dataset\n"+
-                "prediction_feature: '"+prediction_feature.to_s+"'\n"+
-                "test_dataset: '"+test_target_dataset_uri.to_s+"'\n"+
-                "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil
-        else
-          test_target_dataset = OpenTox::Dataset.find test_target_dataset_uri,subjectid
-          raise "test target datset not found: '"+test_target_dataset_uri.to_s+"'" unless test_target_dataset
-          if CHECK_VALUES
-            test_dataset.compounds.each do |c|
-              raise "test compound not found on test class dataset "+c.to_s unless test_target_dataset.compounds.include?(c)
-            end
-          end
-          raise "prediction_feature not found in test_target_dataset\n"+
-                "prediction_feature: '"+prediction_feature.to_s+"'\n"+
-                "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+
-                "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil
-        end
-        
-        @compounds = test_dataset.compounds
-        LOGGER.debug "test dataset size: "+@compounds.size.to_s
-        raise "test dataset is empty "+test_dataset_uri.to_s unless @compounds.size>0
-        
-        if feature_type=="classification"
-          accept_values = test_target_dataset.features[prediction_feature][OT.acceptValue]
-          raise "'"+OT.acceptValue.to_s+"' missing/invalid for feature '"+prediction_feature.to_s+"' in dataset '"+
-            test_target_dataset_uri.to_s+"', acceptValues are: '"+accept_values.inspect+"'" if accept_values==nil or accept_values.length<2 
-        else
-          accept_values=nil
-        end
+        @compounds = []
+        all_predicted_values = []
+        all_actual_values = []
+        all_confidence_values = []
+        accept_values = nil
         
-        actual_values = []
-        @compounds.each do |c|
-          case feature_type
-          when "classification"
-            actual_values << classification_value(test_target_dataset, c, prediction_feature, accept_values)
-          when "regression"
-            actual_values << regression_value(test_target_dataset, c, prediction_feature)
-          end
+        if task
+          task_step = 100 / (test_dataset_uris.size*2 + 1)
+          task_status = 0
         end
-        task.progress(40) if task # loaded actual values
+
+        test_dataset_uris.size.times do |i|
+          
+          test_dataset_uri = test_dataset_uris[i]
+          test_target_dataset_uri = test_target_dataset_uris[i]
+          prediction_dataset_uri = prediction_dataset_uris[i]
+          predicted_variable = predicted_variables[i]
+          
+          predicted_variable=prediction_feature if predicted_variable==nil
         
-        prediction_dataset = OpenTox::Dataset.find prediction_dataset_uri,subjectid
-        raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset
+          test_dataset = Lib::DatasetCache.find test_dataset_uri,subjectid
+          raise "test dataset not found: '"+test_dataset_uri.to_s+"'" unless test_dataset
         
-        # TODO: remove LAZAR_PREDICTION_DATASET_HACK
-        no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil
-        if no_prediction_feature
-          one_entry_per_compound = true
-          @compounds.each do |c|
-            if prediction_dataset.data_entries[c] and prediction_dataset.data_entries[c].size != 1
-              one_entry_per_compound = false
-              break
-            end
-          end
-          msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+
-            prediction_dataset.features.keys.inspect
-          if one_entry_per_compound
-            LOGGER.warn msg
+          if test_target_dataset_uri == nil || test_target_dataset_uri.strip.size==0 || test_target_dataset_uri==test_dataset_uri
+            test_target_dataset_uri = test_dataset_uri
+            test_target_dataset = test_dataset
+            raise "prediction_feature not found in test_dataset, specify a test_target_dataset\n"+
+                  "prediction_feature: '"+prediction_feature.to_s+"'\n"+
+                  "test_dataset: '"+test_target_dataset_uri.to_s+"'\n"+
+                  "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil
           else
-            raise msg
+            test_target_dataset = Lib::DatasetCache.find test_target_dataset_uri,subjectid
+            raise "test target datset not found: '"+test_target_dataset_uri.to_s+"'" unless test_target_dataset
+            if CHECK_VALUES
+              test_dataset.compounds.each do |c|
+                raise "test compound not found on test class dataset "+c.to_s unless test_target_dataset.compounds.include?(c)
+              end
+            end
+            raise "prediction_feature not found in test_target_dataset\n"+
+                  "prediction_feature: '"+prediction_feature.to_s+"'\n"+
+                  "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+
+                  "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil
           end
-        end
-
-        raise "more predicted than test compounds test:"+@compounds.size.to_s+" < prediction:"+
-          prediction_dataset.compounds.size.to_s if @compounds.size < prediction_dataset.compounds.size
-        if CHECK_VALUES
-          prediction_dataset.compounds.each do |c| 
-            raise "predicted compound not found in test dataset:\n"+c+"\ntest-compounds:\n"+
-              @compounds.collect{|c| c.to_s}.join("\n") if @compounds.index(c)==nil
+            
+          compounds = test_dataset.compounds
+          LOGGER.debug "test dataset size: "+compounds.size.to_s
+          raise "test dataset is empty "+test_dataset_uri.to_s unless compounds.size>0
+          
+          if feature_type=="classification"
+            av = test_target_dataset.features[prediction_feature][OT.acceptValue]
+            raise "'"+OT.acceptValue.to_s+"' missing/invalid for feature '"+prediction_feature.to_s+"' in dataset '"+
+              test_target_dataset_uri.to_s+"', acceptValues are: '"+av.inspect+"'" if av==nil or av.length<2
+            if accept_values==nil
+              accept_values=av
+            else
+              raise "accept values (in folds) differ "+av.inspect+" != "+accept_values.inspect if av!=accept_values
+            end
           end
-        end
-        
-        predicted_values = []
-        confidence_values = []
-        @compounds.each do |c|
-          if prediction_dataset.compounds.index(c)==nil
-            predicted_values << nil
-            confidence_values << nil
-          else
+          
+          actual_values = []
+          compounds.each do |c|
             case feature_type
             when "classification"
-              # TODO: remove LAZAR_PREDICTION_DATASET_HACK
-              predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, accept_values)
+              actual_values << classification_value(test_target_dataset, c, prediction_feature, accept_values)
             when "regression"
-              predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable)
+              actual_values << regression_value(test_target_dataset, c, prediction_feature)
+            end
+          end
+          task.progress( task_status += task_step ) if task # loaded actual values
+        
+          prediction_dataset = Lib::DatasetCache.find prediction_dataset_uri,subjectid
+          raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset
+        
+          # TODO: remove LAZAR_PREDICTION_DATASET_HACK
+          no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil
+          if no_prediction_feature
+            one_entry_per_compound = true
+            compounds.each do |c|
+              if prediction_dataset.data_entries[c] and prediction_dataset.data_entries[c].size != 1
+                one_entry_per_compound = false
+                break
+              end
+            end
+            msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+
+              prediction_dataset.features.keys.inspect
+            if one_entry_per_compound
+              LOGGER.warn msg
+            else
+              raise msg
             end
-            # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable)
-            conf = 1
-            begin
-              feature = prediction_dataset.data_entries[c].keys[0]
-              feature_data = prediction_dataset.features[feature]
-              conf = feature_data[OT.confidence] if feature_data[OT.confidence]!=nil 
-            rescue
-              LOGGER.warn "could not get confidence"
+          end
+  
+          raise "more predicted than test compounds test:"+compounds.size.to_s+" < prediction:"+
+            prediction_dataset.compounds.size.to_s if compounds.size < prediction_dataset.compounds.size
+          if CHECK_VALUES
+            prediction_dataset.compounds.each do |c| 
+              raise "predicted compound not found in test dataset:\n"+c+"\ntest-compounds:\n"+
+                compounds.collect{|c| c.to_s}.join("\n") if compounds.index(c)==nil
             end
-            confidence_values << conf
           end
+          
+          predicted_values = []
+          confidence_values = []
+          count = 0
+          compounds.each do |c|
+            if prediction_dataset.compounds.index(c)==nil
+              predicted_values << nil
+              confidence_values << nil
+            else
+              case feature_type
+              when "classification"
+                # TODO: remove LAZAR_PREDICTION_DATASET_HACK
+                predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, accept_values)
+              when "regression"
+                predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable)
+              end
+              # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable)
+              conf = predicted_values[count]!=nil ? 1 : 0 
+              begin
+                feature = prediction_dataset.data_entries[c].keys[0]
+                feature_data = prediction_dataset.features[feature]
+                conf = feature_data[OT.confidence] if feature_data[OT.confidence]!=nil 
+              rescue
+                LOGGER.warn "could not get confidence"
+              end
+              confidence_values << conf
+            end
+            count += 1
+          end
+          
+          @compounds += compounds
+          all_predicted_values += predicted_values
+          all_actual_values += actual_values
+          all_confidence_values += confidence_values
+          
+          task.progress( task_status += task_step ) if task # loaded predicted values and confidence
         end
-        task.progress(80) if task # loaded predicted values and confidence
         
-        super(predicted_values, actual_values, confidence_values, feature_type, accept_values)
-        raise "illegal num compounds "+num_info if  @compounds.size != @predicted_values.size
-        task.progress(100) if task # done with the mathmatics
+      super(all_predicted_values, all_actual_values, all_confidence_values, feature_type, accept_values)
+      raise "illegal num compounds "+num_info if  @compounds.size != @predicted_values.size
+      task.progress(100) if task # done with the mathmatics
     end
     
     private
@@ -205,6 +239,7 @@ module Lib
     def self.to_array( predictions, add_pic=false, format=false )
   
       res = []
+      conf_column = nil
       predictions.each do |p|
         (0..p.num_instances-1).each do |i|
           a = []
@@ -230,13 +265,22 @@ module Lib
             end
           end
           if p.confidence_values_available?
-            a << (format ? p.confidence_value(i).to_nice_s : p.confidence_value(i))
+            conf_column = a.size
+            a << p.confidence_value(i) #(format ? p.confidence_value(i).to_nice_s : p.confidence_value(i))
           end
           a << p.identifier(i)
           res << a
         end
       end
-        
+      
+      if conf_column!=nil
+        res.sort!{ |x,y| y[4] <=> x[4] }
+        if format
+          res.each do |a|
+            a[4] = a[4].to_nice_s
+          end
+        end
+      end
       header = []
       header << "compound" if add_pic
       header << "actual value"
diff --git a/reach_reports/reach_service.rb b/reach_reports/reach_service.rb
index 0cf4172..b6c6350 100755
--- a/reach_reports/reach_service.rb
+++ b/reach_reports/reach_service.rb
@@ -162,33 +162,36 @@ module ReachReports
           next if cvs.size==0
           lmo << "crossvalidation/s on "+desc
           cvs.each do |cv|
-            lmo << "crossvalidation: "+cv.crossvalidation_uri
-            lmo << "dataset (see 9.3 Validation data): "+cv.dataset_uri
-            val_datasets << cv.dataset_uri
-            lmo << "settings: num-folds="+cv.num_folds.to_s+", random-seed="+cv.random_seed.to_s+", stratified:"+cv.stratified.to_s
-
-            val  = YAML.load( OpenTox::RestClientWrapper.get(File.join(cv.crossvalidation_uri,"statistics"),{:subjectid => r.subjectid}) )
-            case feature_type
-            when "classification"
-              lmo << "percent_correct: "+val[OT.classificationStatistics][OT.percentCorrect].to_s
-              lmo << "weighted AUC: "+val[OT.classificationStatistics][OT.weightedAreaUnderRoc].to_s
-            when "regression"
-              lmo << "root_mean_squared_error: "+val[OT.regressionStatistics][OT.rootMeanSquaredError].to_s
-              lmo << "r_square "+val[OT.regressionStatistics][OT.rSquare].to_s
-            end
-            reports = OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"],
-              "report/crossvalidation?crossvalidation_uris="+cv.crossvalidation_uri),{:subjectid => r.subjectid})
-            if reports and reports.chomp.size>0
-              lmo << "for more info see report: "+reports.split("\n")[0]
-            else
-              lmo << "for more info see report: not yet created for '"+cv.crossvalidation_uri+"'"
+            begin
+              lmo << "crossvalidation: "+cv.crossvalidation_uri
+              lmo << "dataset (see 9.3 Validation data): "+cv.dataset_uri
+              val_datasets << cv.dataset_uri
+              lmo << "settings: num-folds="+cv.num_folds.to_s+", random-seed="+cv.random_seed.to_s+", stratified:"+cv.stratified.to_s
+  
+              val  = YAML.load( OpenTox::RestClientWrapper.get(File.join(cv.crossvalidation_uri,"statistics"),{:subjectid => r.subjectid}) )
+              case feature_type
+              when "classification"
+                lmo << "percent_correct: "+val[OT.classificationStatistics][OT.percentCorrect].to_s
+                lmo << "weighted AUC: "+val[OT.classificationStatistics][OT.weightedAreaUnderRoc].to_s
+              when "regression"
+                lmo << "root_mean_squared_error: "+val[OT.regressionStatistics][OT.rootMeanSquaredError].to_s
+                lmo << "r_square "+val[OT.regressionStatistics][OT.rSquare].to_s
+              end
+              reports = OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"],
+                "report/crossvalidation?crossvalidation_uris="+cv.crossvalidation_uri),{:subjectid => r.subjectid})
+              if reports and reports.chomp.size>0
+                lmo << "for more info see report: "+reports.split("\n")[0]
+              else
+                lmo << "for more info see report: not yet created for '"+cv.crossvalidation_uri+"'"
+              end
+            rescue => ex
+              LOGGER.warn "could not add cv "+cv.crossvalidation_uri+" : "+ex.message            
             end
           end
           lmo << ""
         end
       end
-      
-    else
+   else
       lmo = [ "no prediction algortihm for model found, crossvalidation not possible" ]
     end
     r.qsar_robustness.lmo = lmo.to_html
diff --git a/report/plot_factory.rb b/report/plot_factory.rb
index a4e415a..b7c920a 100644
--- a/report/plot_factory.rb
+++ b/report/plot_factory.rb
@@ -354,7 +354,7 @@ module Reports
       c = roc_values[:confidence_values]
       p = roc_values[:predicted_values]
       a = roc_values[:actual_values]
-      raise "no prediction values for roc-plot" if p.size==0
+      raise "no prediction values for confidence plot" if p.size==0
      
       (0..p.size-2).each do |i|
         ((i+1)..p.size-1).each do |j|
diff --git a/report/report_content.rb b/report/report_content.rb
index ca04f25..cc4c13c 100755
--- a/report/report_content.rb
+++ b/report/report_content.rb
@@ -31,11 +31,11 @@ class Reports::ReportContent
     level = 0.90                       
     test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, 
       group_attribute, test_attribute, "paired_ttest", level )
-    puts test_matrix.inspect
+    #puts test_matrix.inspect
     titles = test_matrix[:titles]
     matrix = test_matrix[:matrix]
     table = []
-    puts titles.inspect
+    #puts titles.inspect
     table << [""] + titles
     titles.size.times do |i|
       table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") }
@@ -47,10 +47,10 @@ class Reports::ReportContent
   end
   
   def add_predictions( validation_set, 
-                              validation_attributes=[],
-                              section_title="Predictions",
-                              section_text=nil,
-                              table_title="Predictions")
+                        validation_attributes=[],
+                        section_title="Predictions",
+                        section_text=nil,
+                        table_title="Predictions")
 
     #PENING
     raise "validation attributes not implemented in get prediction array" if  validation_attributes.size>0
@@ -109,7 +109,7 @@ class Reports::ReportContent
     
     if (search_for_existing_report_type)
       vals.size.times do |i|
-        puts i
+        #puts i
         if (i==0)
           vals[i] = [ "Reports" ] + vals[i]
           puts vals[i].inspect
diff --git a/report/validation_access.rb b/report/validation_access.rb
index e9b6e19..d0c3a1d 100755
--- a/report/validation_access.rb
+++ b/report/validation_access.rb
@@ -25,7 +25,7 @@ class Reports::ValidationDB
         raise OpenTox::NotFoundError.new "crossvalidation with id "+cv_id.to_s+" not found" unless cv
         raise OpenTox::BadRequestError.new("crossvalidation with id '"+cv_id.to_s+"' not finished") unless cv.finished
         #res += Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s}
-        res += Validation::Validation.find( :crossvalidation_id => cv_id ).collect{|v| v.validation_uri.to_s }
+        res += Validation::Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|v| v.validation_uri.to_s }
       else
         res += [u.to_s]
       end
@@ -35,7 +35,7 @@ class Reports::ValidationDB
   
   def init_validation(validation, uri, subjectid=nil)
   
-    raise OpenTox::BadRequestError.new "not a validation uri: "+uri.to_s unless uri =~ /.*\/[0-9]+/
+    raise OpenTox::BadRequestError.new "not a validation uri: "+uri.to_s unless uri =~ /\/[0-9]+$/
     validation_id = uri.split("/")[-1]
     raise OpenTox::BadRequestError.new "invalid validation id "+validation_id.to_s unless validation_id!=nil and 
       (validation_id.to_i > 0 || validation_id.to_s=="0" )
@@ -56,6 +56,31 @@ class Reports::ValidationDB
       subset_props.each{ |prop| validation.send("#{prop.to_s}=".to_sym, subset[prop]) } if subset
     end
   end
+  
+  def init_validation_from_cv_statistics( validation, cv_uri, subjectid=nil )
+    
+    raise OpenTox::BadRequestError.new "not a crossvalidation uri: "+cv_uri.to_s unless cv_uri.uri? and cv_uri =~ /crossvalidation.*\/[0-9]+$/
+    cv_id = cv_uri.split("/")[-1]
+    raise OpenTox::NotAuthorizedError.new "Not authorized: GET "+cv_uri.to_s if
+      AA_SERVER and !OpenTox::Authorization.authorized?(cv_uri,"GET",subjectid)
+    cv = Validation::Crossvalidation.get(cv_id)
+    raise OpenTox::NotFoundError.new "crossvalidation with id "+crossvalidation_id.to_s+" not found" unless cv
+    raise OpenTox::BadRequestError.new "crossvalidation with id "+crossvalidation_id.to_s+" is not finished yet" unless cv.finished
+    v = Validation::Validation.from_cv_statistics(cv_id, subjectid)
+    (Validation::VAL_PROPS + Validation::VAL_CV_PROPS).each do |p|
+      validation.send("#{p.to_s}=".to_sym, v.send(p))
+    end
+    {:classification_statistics => Validation::VAL_CLASS_PROPS, 
+     :regression_statistics => Validation::VAL_REGR_PROPS}.each do |subset_name,subset_props|
+      subset = v.send(subset_name)
+      subset_props.each{ |prop| validation.send("#{prop.to_s}=".to_sym, subset[prop]) } if subset
+    end
+    #cv props
+    Validation::CROSS_VAL_PROPS.each do |p|
+      validation.send("#{p.to_s}=".to_sym, cv.send(p.to_s))
+    end
+    validation.crossvalidation_uri = cv_uri
+  end
     
   def init_cv(validation)
     
@@ -76,7 +101,8 @@ class Reports::ValidationDB
   
   def get_accept_values( validation, subjectid=nil )
     # PENDING So far, one has to load the whole dataset to get the accept_value from ambit
-    d = OpenTox::Dataset.find( validation.test_target_dataset_uri, subjectid )
+    d = Lib::DatasetCache.find( validation.test_target_dataset_uri, subjectid )
+    raise "cannot get test target dataset for accept values, dataset: "+validation.test_target_dataset_uri.to_s unless d
     accept_values = d.features[validation.prediction_feature][OT.acceptValue]
     raise "cannot get accept values from dataset "+validation.test_target_dataset_uri.to_s+" for feature "+
       validation.prediction_feature+":\n"+d.features[validation.prediction_feature].to_yaml unless accept_values!=nil
diff --git a/report/validation_data.rb b/report/validation_data.rb
index 42b179b..11fa737 100755
--- a/report/validation_data.rb
+++ b/report/validation_data.rb
@@ -81,6 +81,12 @@ module Reports
       @subjectid = subjectid
       #raise "subjectid is nil" unless subjectid
     end
+    
+    def self.from_cv_statistics( cv_uri, subjectid = nil )
+      v = ReportValidation.new(nil, subjectid)
+      @@validation_access.init_validation_from_cv_statistics(v, cv_uri, subjectid)
+      v
+    end
   
     # returns/creates predictions, cache to save rest-calls/computation time
     #
@@ -409,17 +415,30 @@ module Reports
       #compute grouping
       grouping = Util.group(@validations, equal_attributes)
       #puts "groups "+grouping.size.to_s
-  
-      Lib::MergeObjects.register_merge_attributes( ReportValidation,
-        Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL) unless 
-          Lib::MergeObjects.merge_attributes_registered?(ReportValidation)
-  
-      #merge
-      grouping.each do |g|
-        new_set.validations.push(g[0].clone_validation)
-        g[1..-1].each do |v|
-          new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v)
+
+      if ( equal_attributes.include?(:crossvalidation_id) )
+        # do not merge, use crossvalidation statistics
+        raise "statistics vs merging problem" if equal_attributes.size!=1
+        grouping.each do |g|
+          new_set.validations << ReportValidation.from_cv_statistics(g[0].crossvalidation_uri)
         end
+      else
+        #merge
+        Lib::MergeObjects.register_merge_attributes( ReportValidation,
+          Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL) unless 
+            Lib::MergeObjects.merge_attributes_registered?(ReportValidation)
+        grouping.each do |g|
+          new_set.validations << g[0].clone_validation
+          w = 1
+          g[1..-1].each do |v|
+            new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v,w,1)
+            w+=1
+          end
+        end
+      end
+      
+      new_set.validations.each do |v|
+        raise "not a validation "+v.class.to_s+" "+v.to_s unless v.is_a?(Reports::ReportValidation)
       end
       
       return new_set
diff --git a/validation/validation_application.rb b/validation/validation_application.rb
index 4bcd07d..7db2a6a 100755
--- a/validation/validation_application.rb
+++ b/validation/validation_application.rb
@@ -3,8 +3,7 @@
   require lib
 end
 
-require 'lib/merge.rb'
-#require 'lib/active_record_setup.rb'
+require 'lib/dataset_cache.rb'
 require 'validation/validation_service.rb'
 
 get '/crossvalidation/?' do
@@ -41,6 +40,8 @@ post '/crossvalidation/?' do
     cv = Validation::Crossvalidation.create cv_params
     cv.subjectid = @subjectid
     cv.perform_cv( params[:prediction_feature], params[:algorithm_params], task )
+    # computation of stats is cheap as dataset are already loaded into the memory
+    Validation::Validation.from_cv_statistics( cv.id, @subjectid )
     cv.crossvalidation_uri
   end
   return_task(task)
@@ -108,33 +109,9 @@ get '/crossvalidation/:id' do
 end
 
 get '/crossvalidation/:id/statistics' do
-  LOGGER.info "get merged validation-result for crossvalidation with id "+params[:id].to_s
-#  begin
-    #crossvalidation = Validation::Crossvalidation.find(params[:id])
-#  rescue ActiveRecord::RecordNotFound => ex
-#    raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found."
-#  end
-  #crossvalidation = Validation::Crossvalidation.find(params[:id])
-  crossvalidation = Validation::Crossvalidation.get(params[:id])
-  
-  raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found." unless crossvalidation
-  raise OpenTox::BadRequestError.new "Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished
-  
-  Lib::MergeObjects.register_merge_attributes( Validation::Validation,
-    Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL-[:date,:validation_uri,:crossvalidation_uri]) unless 
-      Lib::MergeObjects.merge_attributes_registered?(Validation::Validation)
-  
-  #v = Lib::MergeObjects.merge_array_objects( Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) )
-  # convert ohm:set into array, as ohm:set[0]=nil(!)
-  vals = Validation::Validation.find( :crossvalidation_id => params[:id] ).collect{|x| x}
-#  LOGGER.debug vals.collect{|v| v.validation_uri}.join("\n")
-#  LOGGER.debug vals.size
-#  LOGGER.debug vals.class
-  raise "could not load all validations for crossvalidation" if vals.include?(nil)
-  v = Lib::MergeObjects.merge_array_objects( vals )
-  v.date = nil
-  #v.id = nil
   
+  LOGGER.info "get crossvalidation statistics for crossvalidation with id "+params[:id].to_s
+  v = Validation::Validation.from_cv_statistics( params[:id], @subjectid )
   case request.env['HTTP_ACCEPT'].to_s
   when /text\/html/
     related_links = 
@@ -187,7 +164,7 @@ get '/crossvalidation/:id/predictions' do
   raise OpenTox::BadRequestError.new "Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished
   
   content_type "application/x-yaml"
-  validations = Validation::Validation.find( :crossvalidation_id => params[:id] )
+  validations = Validation::Validation.find( :crossvalidation_id => params[:id], :validation_type => "crossvalidation" )
   p = Lib::OTPredictions.to_array( validations.collect{ |v| v.compute_validation_stats_with_model(nil, true) } ).to_yaml
   
   case request.env['HTTP_ACCEPT'].to_s
diff --git a/validation/validation_format.rb b/validation/validation_format.rb
index 6fdea61..23b1996 100755
--- a/validation/validation_format.rb
+++ b/validation/validation_format.rb
@@ -83,7 +83,7 @@ module Validation
       end
       v = []
       #Validation.find( :all, :conditions => { :crossvalidation_id => self.id } ).each do |val|
-      Validation.find( :crossvalidation_id => self.id ).each do |val|
+      Validation.find( :crossvalidation_id => self.id, :validation_type => "crossvalidation" ).each do |val|
         v.push( val.validation_uri.to_s )
       end
       h[:validation_uris] = v
diff --git a/validation/validation_service.rb b/validation/validation_service.rb
index dcfb8d7..99d8672 100755
--- a/validation/validation_service.rb
+++ b/validation/validation_service.rb
@@ -31,15 +31,49 @@ end
 module Validation
   
   class Validation
-      
-    # constructs a validation object, Rsets id und uri
-    #def initialize( params={} )
-      #raise "do not set id manually" if params[:id]
-      #params[:finished] = false
-      #super params
-      #self.save!
-      #raise "internal error, validation-id not set "+to_yaml if self.id==nil
-    #end
+    
+    def self.from_cv_statistics( cv_id, subjectid=nil )
+      v =  Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation_statistics" ).first
+      unless v
+        crossvalidation = Crossvalidation.get(cv_id)
+        raise OpenTox::NotFoundError.new "Crossvalidation '#{cv_id}' not found." unless crossvalidation
+        raise OpenTox::BadRequestError.new "Crossvalidation '"+cv_id.to_s+"' not finished" unless crossvalidation.finished
+        
+        vals = Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|x| x}
+        feature_type = OpenTox::Model::Generic.new(vals.first.model_uri).feature_type(@subjectid)
+        test_dataset_uris = vals.collect{|v| v.test_dataset_uri}
+        test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri}
+        prediction_feature = vals.first.prediction_feature
+        prediction_dataset_uris = vals.collect{|v| v.prediction_dataset_uri}
+        predicted_variables = vals.collect{|v| nil}
+        prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, 
+          prediction_dataset_uris, predicted_variables, @subjectid )
+          
+        v = Validation.new
+        case feature_type
+        when "classification"
+          v.classification_statistics = prediction.compute_stats
+        when "regression"
+          v.regression_statistics = prediction.compute_stats
+        end
+        v.update :num_instances => prediction.num_instances,
+               :num_without_class => prediction.num_without_class,
+               :percent_without_class => prediction.percent_without_class,
+               :num_unpredicted => prediction.num_unpredicted,
+               :percent_unpredicted => prediction.percent_unpredicted,
+               :finished => true
+        (VAL_PROPS_GENERAL-[:validation_uri]).each do |p|
+          v.send("#{p.to_s}=".to_sym, vals.collect{ |vv| vv.send(p) }.uniq.join(","))
+        end
+        v.date = crossvalidation.date
+        v.validation_type = "crossvalidation_statistics"
+        v.crossvalidation_id = crossvalidation.id
+        v.crossvalidation_fold = vals.collect{ |vv| vv.crossvalidation_fold }.uniq.join(",")       
+        v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(",")
+        v.save
+      end
+      v
+    end
     
     # deletes a validation
     # PENDING: model and referenced datasets are deleted as well, keep it that way?
@@ -238,21 +272,7 @@ module Validation
   
   class Crossvalidation
     
-    # constructs a crossvalidation, id and uri are set
-    #def initialize( params={} )
-    #  
-    #  raise "do not set id manually" if params[:id]
-    #  params[:num_folds] = 10 if params[:num_folds]==nil
-    #  params[:random_seed] = 1 if params[:random_seed]==nil
-    #  params[:stratified] = false if params[:stratified]==nil
-    #  params[:finished] = false
-    #  super params
-    #  self.save!
-    #  raise "internal error, crossvalidation-id not set" if self.id==nil
-    #end
-    
     def perform_cv ( prediction_feature, algorithm_params=nil, task=nil )
-      
       create_cv_datasets( prediction_feature, OpenTox::SubTask.create(task, 0, 33) )
       perform_cv_validations( algorithm_params, OpenTox::SubTask.create(task, 33, 100) )
     end
@@ -324,7 +344,7 @@ module Validation
       cvs.each do |cv|
         next if AA_SERVER and !OpenTox::Authorization.authorized?(cv.crossvalidation_uri,"GET",self.subjectid)
         tmp_val = []
-        Validation.find( :crossvalidation_id => cv.id ).each do |v|
+        Validation.find( :crossvalidation_id => cv.id, :validation_type => "crossvalidation" ).each do |v|
           break unless 
             v.prediction_feature == prediction_feature and
             OpenTox::Dataset.exist?(v.training_dataset_uri,self.subjectid) and 
@@ -353,7 +373,7 @@ module Validation
     # stores uris in validation objects 
     def create_new_cv_datasets( prediction_feature, task = nil )
       LOGGER.debug "creating datasets for crossvalidation"
-      orig_dataset = OpenTox::Dataset.find(self.dataset_uri,self.subjectid)
+      orig_dataset = Lib::DatasetCache.find(self.dataset_uri,self.subjectid)
       raise OpenTox::NotFoundError.new "Dataset not found: "+self.dataset_uri.to_s unless orig_dataset
       
       shuffled_compounds = orig_dataset.compounds.shuffle( self.random_seed )
@@ -465,7 +485,7 @@ module Validation
       
       random_seed=1 unless random_seed
       
-      orig_dataset = OpenTox::Dataset.find orig_dataset_uri,subjectid
+      orig_dataset = Lib::DatasetCache.find orig_dataset_uri,subjectid
       orig_dataset.load_all
       raise OpenTox::NotFoundError.new "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset
       if prediction_feature
@@ -530,7 +550,7 @@ module Validation
       task.progress(100) if task
       
       if ENV['RACK_ENV'] =~ /test|debug/
-        training_dataset = OpenTox::Dataset.find result[:training_dataset_uri],subjectid
+        training_dataset = Lib::DatasetCache.find result[:training_dataset_uri],subjectid
         raise OpenTox::NotFoundError.new "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless training_dataset
         training_dataset.load_all
         value_count = 0
@@ -539,7 +559,7 @@ module Validation
         end
         raise  "training compounds error" unless value_count==training_compounds.size
         raise OpenTox::NotFoundError.new "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless 
-          OpenTox::Dataset.find result[:test_dataset_uri], subjectid
+          Lib::DatasetCache.find result[:test_dataset_uri], subjectid
       end
       LOGGER.debug "bootstrapping done, training dataset: '"+result[:training_dataset_uri].to_s+"', test dataset: '"+result[:test_dataset_uri].to_s+"'"
       
@@ -554,7 +574,7 @@ module Validation
       random_seed=1 unless random_seed
       random_seed = random_seed.to_i
       
-      orig_dataset = OpenTox::Dataset.find orig_dataset_uri, subjectid
+      orig_dataset = Lib::DatasetCache.find orig_dataset_uri, subjectid
       orig_dataset.load_all subjectid
       raise OpenTox::NotFoundError.new "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset
       raise OpenTox::NotFoundError.new "Split ratio invalid: "+split_ratio.to_s unless split_ratio and split_ratio=split_ratio.to_f
@@ -597,7 +617,7 @@ module Validation
         subjectid ).uri
       task.progress(66) if task
 
-#      d = OpenTox::Dataset.find(result[:training_dataset_uri])
+#      d = Lib::DatasetCache.find(result[:training_dataset_uri])
 #      d.data_entries.values.each do |v|
 #        puts v.inspect
 #        puts v.values[0].to_s+" "+v.values[0].class.to_s
@@ -617,8 +637,8 @@ module Validation
       
       if ENV['RACK_ENV'] =~ /test|debug/
         raise OpenTox::NotFoundError.new "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless 
-          OpenTox::Dataset.find(result[:training_dataset_uri],subjectid)
-        test_data = OpenTox::Dataset.find result[:test_dataset_uri],subjectid
+          Lib::DatasetCache.find(result[:training_dataset_uri],subjectid)
+        test_data = Lib::DatasetCache.find result[:test_dataset_uri],subjectid
         raise OpenTox::NotFoundError.new "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless test_data 
         test_data.load_compounds subjectid
         raise "Test dataset num coumpounds != "+(compounds.size-split-1).to_s+", instead: "+