new stratified type super added to crossvalidation and traning-test-split validation, add some more metadata to crossvaldiation, add validation_uri to predictions in crossvaldiation-report

6 files changed, 163 insertions, 154 deletions

diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb
index 3be845b..2752fcc 100755
--- a/lib/ot_predictions.rb
+++ b/lib/ot_predictions.rb
@@ -35,7 +35,7 @@ module Lib
       OTPredictions.to_array( [self] )
     end
     
-    def self.to_array( predictions, add_pic=false, format=false )
+    def self.to_array( predictions, add_pic=false, format=false, validation_uris=nil )
   
       confidence_available = false
       predictions.each do |p|
@@ -43,7 +43,10 @@ module Lib
       end
       res = []
       conf_column = nil
+      count = 0
       predictions.each do |p|
+        v_uris = validation_uris[count] if validation_uris
+        count += 1
         (0..p.num_instances-1).each do |i|
           a = []
           
@@ -75,6 +78,9 @@ module Lib
             conf_column = a.size if conf_column==nil
             a << p.confidence_value(i)
           end
+          if validation_uris
+            a << v_uris[i]
+          end
           a << p.identifier(i)
           res << a
         end
@@ -90,12 +96,13 @@ module Lib
         end
       end
       header = []
-      header << "compound" if add_pic
-      header << "actual value"
-      header << "predicted value"
-      header << "classification" if predictions[0].feature_type=="classification"
-      header << "confidence value" if predictions[0].confidence_values_available?
-      header << "compound-uri"
+      header << "Compound" if add_pic
+      header << "Actual value"
+      header << "Predicted value"
+      header << "Classification" if predictions[0].feature_type=="classification"
+      header << "Confidence value" if predictions[0].confidence_values_available?
+      header << "Validation URI" if validation_uris
+      header << "Compound URI"
       res.insert(0, header)
       
       return res
diff --git a/lib/validation_db.rb b/lib/validation_db.rb
index 7d83966..086853e 100755
--- a/lib/validation_db.rb
+++ b/lib/validation_db.rb
@@ -6,8 +6,9 @@ require "lib/merge.rb"
 
 module Validation
 
-  VAL_PROPS_GENERAL = [ :validation_uri, :validation_type, :model_uri, :algorithm_uri, :training_dataset_uri, :prediction_feature,
-                :test_dataset_uri, :test_target_dataset_uri, :prediction_dataset_uri, :date ] 
+  VAL_PROPS_GENERAL = [ :validation_uri, :validation_type, :model_uri, :algorithm_uri, :algorithm_params,
+                :training_dataset_uri, :prediction_feature, :test_dataset_uri, :test_target_dataset_uri, 
+                :prediction_dataset_uri, :date ] 
   VAL_PROPS_SUM = [ :num_instances, :num_without_class, :num_unpredicted ]
   VAL_PROPS_AVG = [:real_runtime, :percent_without_class, :percent_unpredicted ]
   VAL_PROPS = VAL_PROPS_GENERAL + VAL_PROPS_SUM + VAL_PROPS_AVG
@@ -41,7 +42,8 @@ module Validation
     :weighted_mean_absolute_error, :weighted_root_mean_squared_error, :concordance_correlation_coefficient ]
   
   CROSS_VAL_PROPS = [:dataset_uri, :num_folds, :stratified, :random_seed]
-  CROSS_VAL_PROPS_REDUNDANT = [:crossvalidation_uri, :algorithm_uri, :date] + CROSS_VAL_PROPS 
+  CROSS_VAL_PROPS_REDUNDANT = [:crossvalidation_uri, :algorithm_uri, :algorithm_params, 
+    :prediction_feature, :date] + CROSS_VAL_PROPS 
   
   ALL_PROPS = VAL_PROPS + VAL_CV_PROPS + VAL_CLASS_PROPS + VAL_REGR_PROPS + CROSS_VAL_PROPS
 
@@ -55,6 +57,7 @@ module Validation
     attribute :validation_type
     attribute :model_uri
     attribute :algorithm_uri
+    attribute :algorithm_params
     attribute :training_dataset_uri
     attribute :test_target_dataset_uri
     attribute :test_dataset_uri
@@ -77,6 +80,11 @@ module Validation
     index :model_uri
     index :validation_type
     index :crossvalidation_id
+    index :algorithm_uri
+    index :algorithm_params
+    index :prediction_feature
+    index :training_dataset_uri
+    index :test_dataset_uri
     
     attr_accessor :subjectid
     
@@ -141,6 +149,8 @@ module Validation
   class Crossvalidation < Ohm::Model
   
     attribute :algorithm_uri
+    attribute :algorithm_params
+    attribute :prediction_feature
     attribute :dataset_uri
     attribute :date
     attribute :num_folds
@@ -152,6 +162,8 @@ module Validation
     attr_accessor :subjectid
     
     index :algorithm_uri
+    index :algorithm_params
+    index :prediction_feature
     index :dataset_uri
     index :num_folds
     index :random_seed
diff --git a/report/report_content.rb b/report/report_content.rb
index 80473c5..033b367 100755
--- a/report/report_content.rb
+++ b/report/report_content.rb
@@ -63,20 +63,17 @@ class Reports::ReportContent
     end
   end
   
-  def add_predictions( validation_set, 
-                        validation_attributes=[],
+  def add_predictions( validation_set,
+                        add_validation_uris,   
                         section_title="Predictions",
                         section_text=nil,
                         table_title="Predictions")
-
-    #PENING
-    raise "validation attributes not implemented in get prediction array" if  validation_attributes.size>0
-    
     section_table = @xml_report.add_section(@current_section, section_title)
     if validation_set.validations[0].get_predictions
       @xml_report.add_paragraph(section_table, section_text) if section_text
+      v_uris = validation_set.validations.collect{|v| Array.new(v.num_instances.to_i,v.validation_uri)} if add_validation_uris
       @xml_report.add_table(section_table, table_title, Lib::OTPredictions.to_array(validation_set.validations.collect{|v| v.get_predictions}, 
-        true, true))
+        true, true, v_uris))
     else
       @xml_report.add_paragraph(section_table, "No prediction info available.")
     end
diff --git a/report/report_factory.rb b/report/report_factory.rb
index f51b999..f73ffd9 100755
--- a/report/report_factory.rb
+++ b/report/report_factory.rb
@@ -124,7 +124,7 @@ module Reports::ReportFactory
     report.end_section
 
     report.add_result(validation_set, Validation::ALL_PROPS, "All Results", "All Results")
-    report.add_predictions( validation_set )
+    report.add_predictions( validation_set, false )
     task.progress(100) if task
     report
   end
@@ -200,7 +200,7 @@ module Reports::ReportFactory
       
     report.add_result(validation_set, Validation::ALL_PROPS, "All Results", "All Results") if
       (cv_set.unique_value(:num_folds).to_i < cv_set.unique_value(:num_instances).to_i)
-    report.add_predictions( validation_set ) #, [:crossvalidation_fold] )
+    report.add_predictions( validation_set, true )
     task.progress(100) if task
     report
   end
diff --git a/validation/validation_application.rb b/validation/validation_application.rb
index 60fc7df..1bc55f6 100755
--- a/validation/validation_application.rb
+++ b/validation/validation_application.rb
@@ -6,8 +6,16 @@ end
 require 'lib/dataset_cache.rb'
 require 'validation/validation_service.rb'
 
+helpers do
+  def check_stratified(params)
+    params[:stratified] = "false" unless params[:stratified]
+    raise OpenTox::BadRequestError.new "stratified != true|false|super, is #{params[:stratified]}" unless
+      params[:stratified]=~/true|false|super/
+  end
+end
+
 get '/crossvalidation/?' do
-  LOGGER.info "list all crossvalidations"
+  LOGGER.info "list all crossvalidations "+params.inspect
   model_uri = params.delete("model") || params.delete("model_uri")
   if model_uri
     model = OpenTox::Model::Generic.find(model_uri, @subjectid)
@@ -46,17 +54,20 @@ post '/crossvalidation/?' do
   raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0
   raise OpenTox::BadRequestError.new "illegal param-value num_folds: '"+params[:num_folds].to_s+"', must be integer >1" unless params[:num_folds]==nil or 
     params[:num_folds].to_i>1
-    
+  check_stratified(params)
+  
   task = OpenTox::Task.create( "Perform crossvalidation", url_for("/crossvalidation", :full) ) do |task| #, params
     cv_params = { :dataset_uri => params[:dataset_uri],  
                   :algorithm_uri => params[:algorithm_uri],
+                  :algorithm_params => params[:algorithm_params],
+                  :prediction_feature => params[:prediction_feature],
+                  :stratified => params[:stratified],
                   :loo => "false",
                   :subjectid => @subjectid }
     [ :num_folds, :random_seed ].each{ |sym| cv_params[sym] = params[sym] if params[sym] }
-    cv_params[:stratified] = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified]
     cv = Validation::Crossvalidation.create cv_params
     cv.subjectid = @subjectid
-    cv.perform_cv( params[:prediction_feature], params[:algorithm_params], OpenTox::SubTask.create(task,0,95))
+    cv.perform_cv( OpenTox::SubTask.create(task,0,95) )
     # computation of stats is cheap as dataset are already loaded into the memory
     Validation::Validation.from_cv_statistics( cv.id, @subjectid, OpenTox::SubTask.create(task,95,100) )
     cv.crossvalidation_uri
@@ -87,14 +98,16 @@ post '/crossvalidation/loo/?' do
   raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri].to_s.size>0
   raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0
   raise OpenTox::BadRequestError.new "illegal param: num_folds, stratified, random_seed not allowed for loo-crossvalidation" if params[:num_folds] or 
-    params[:stratifed] or params[:random_seed]
+    params[:stratified] or params[:random_seed]
   task = OpenTox::Task.create( "Perform loo-crossvalidation", url_for("/crossvalidation/loo", :full) ) do |task| #, params
-    cv_params = { :dataset_uri => params[:dataset_uri],  
+    cv_params = { :dataset_uri => params[:dataset_uri],
+                  :algorithm_params => params[:algorithm_params],
+                  :prediction_feature => params[:prediction_feature],  
                   :algorithm_uri => params[:algorithm_uri],
                   :loo => "true" }
     cv = Validation::Crossvalidation.create cv_params
     cv.subjectid = @subjectid
-    cv.perform_cv( params[:prediction_feature], params[:algorithm_params], OpenTox::SubTask.create(task,0,95))
+    cv.perform_cv( OpenTox::SubTask.create(task,0,95))
     # computation of stats is cheap as dataset are already loaded into the memory
     Validation::Validation.from_cv_statistics( cv.id, @subjectid, OpenTox::SubTask.create(task,95,100) )
     cv.clean_loo_files( !(params[:algorithm_params] && params[:algorithm_params] =~ /feature_dataset_uri/) )
@@ -344,12 +357,13 @@ post '/training_test_validation/?' do
     task = OpenTox::Task.create( "Perform training-test-validation", url_for("/", :full) ) do |task| #, params
       v = Validation::Validation.create :validation_type => "training_test_validation", 
                         :algorithm_uri => params[:algorithm_uri],
+                        :algorithm_params => params[:algorithm_params],
                         :training_dataset_uri => params[:training_dataset_uri], 
                         :test_dataset_uri => params[:test_dataset_uri],
                         :test_target_dataset_uri => params[:test_target_dataset_uri],
                         :prediction_feature => params[:prediction_feature]
       v.subjectid = @subjectid
-      v.validate_algorithm( params[:algorithm_params], task ) 
+      v.validate_algorithm( task ) 
       v.validation_uri
     end
     return_task(task)
@@ -403,10 +417,11 @@ post '/bootstrapping' do
                      :test_target_dataset_uri => params[:dataset_uri],
                      :prediction_feature => params[:prediction_feature],
                      :algorithm_uri => params[:algorithm_uri],
+                     :algorithm_params => params[:algorithm_params],
                      :training_dataset_uri => params[:training_dataset_uri], 
                      :test_dataset_uri => params[:test_dataset_uri]
     v.subjectid = @subjectid
-    v.validate_algorithm( params[:algorithm_params], OpenTox::SubTask.create(task,33,100))
+    v.validate_algorithm( OpenTox::SubTask.create(task,33,100))
     v.validation_uri
   end
   return_task(task)
@@ -453,18 +468,19 @@ post '/training_test_split' do
   raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri].to_s.size>0
   raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri].to_s.size>0
   raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0
+  check_stratified(params)
   task = OpenTox::Task.create( "Perform training test split validation", url_for("/training_test_split", :full) )  do |task| #, params
-    strat = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified]
     params.merge!( Validation::Util.train_test_dataset_split(params[:dataset_uri], params[:prediction_feature], 
-      @subjectid,  strat, params[:split_ratio], params[:random_seed], OpenTox::SubTask.create(task,0,33)))
+      @subjectid,  params[:stratified], params[:split_ratio], params[:random_seed], OpenTox::SubTask.create(task,0,33)))
     v = Validation::Validation.create  :validation_type => "training_test_split", 
                      :training_dataset_uri => params[:training_dataset_uri], 
                      :test_dataset_uri => params[:test_dataset_uri],
                      :test_target_dataset_uri => params[:dataset_uri],
                      :prediction_feature => params[:prediction_feature],
-                     :algorithm_uri => params[:algorithm_uri]
+                     :algorithm_uri => params[:algorithm_uri],
+                     :algorithm_params => params[:algorithm_params]
     v.subjectid = @subjectid
-    v.validate_algorithm( params[:algorithm_params], OpenTox::SubTask.create(task,33,100))
+    v.validate_algorithm( OpenTox::SubTask.create(task,33,100))
     v.validation_uri
   end
   return_task(task)
@@ -546,10 +562,10 @@ end
 post '/plain_training_test_split' do
   LOGGER.info "creating pure training test split "+params.inspect
   raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri]
+  check_stratified(params)
   task = OpenTox::Task.create( "Create data-split", url_for("/plain_training_test_split", :full) ) do |task|
-    strat = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified]
     result = Validation::Util.train_test_dataset_split(params[:dataset_uri], params[:prediction_feature], @subjectid,
-       strat, params[:split_ratio], params[:random_seed])
+      params[:stratified], params[:split_ratio], params[:random_seed], task)
     content_type "text/uri-list"
     result[:training_dataset_uri]+"\n"+result[:test_dataset_uri]+"\n"
   end
diff --git a/validation/validation_service.rb b/validation/validation_service.rb
index 686a287..c433161 100755
--- a/validation/validation_service.rb
+++ b/validation/validation_service.rb
@@ -111,7 +111,7 @@ module Validation
     end
     
     # validates an algorithm by building a model and validating this model
-    def validate_algorithm( algorithm_params=nil, task=nil )
+    def validate_algorithm( task=nil )
       raise "validation_type missing" unless self.validation_type
       raise OpenTox::BadRequestError.new "no algorithm uri: '"+self.algorithm_uri.to_s+"'" if self.algorithm_uri==nil or self.algorithm_uri.to_s.size<1
       
@@ -301,9 +301,9 @@ module Validation
   
   class Crossvalidation
     
-    def perform_cv ( prediction_feature, algorithm_params=nil, task=nil )
-      create_cv_datasets( prediction_feature, OpenTox::SubTask.create(task, 0, 33) )
-      perform_cv_validations( algorithm_params, OpenTox::SubTask.create(task, 33, 100) )
+    def perform_cv ( task=nil )
+      create_cv_datasets( OpenTox::SubTask.create(task, 0, 33) )
+      perform_cv_validations( OpenTox::SubTask.create(task, 33, 100) )
     end
     
     def clean_loo_files( delete_feature_datasets )
@@ -349,27 +349,27 @@ module Validation
     end
     
     # creates the cv folds
-    def create_cv_datasets( prediction_feature, task=nil )
+    def create_cv_datasets( task=nil )
       if self.loo=="true"
         orig_dataset = Lib::DatasetCache.find(self.dataset_uri,self.subjectid)
         self.num_folds = orig_dataset.compounds.size
         self.random_seed = 0
-        self.stratified = false
+        self.stratified = "false"
       else
         self.random_seed = 1 unless self.random_seed
         self.num_folds = 10 unless self.num_folds
-        self.stratified = false unless self.stratified
+        self.stratified = "false" unless self.stratified
       end
-      if copy_cv_datasets( prediction_feature )
+      if copy_cv_datasets()
         # dataset folds of a previous crossvalidaiton could be used 
         task.progress(100) if task
       else
-        create_new_cv_datasets( prediction_feature, task )
+        create_new_cv_datasets( task )
       end
     end
     
     # executes the cross-validation (build models and validates them)
-    def perform_cv_validations( algorithm_params, task=nil )
+    def perform_cv_validations( task=nil )
       
       LOGGER.debug "perform cv validations "+algorithm_params.inspect
       i = 0
@@ -377,8 +377,7 @@ module Validation
       @tmp_validations.each do | val |
         validation = Validation.create val
         validation.subjectid = self.subjectid
-        validation.validate_algorithm( algorithm_params, 
-          OpenTox::SubTask.create(task, i * task_step, ( i + 1 ) * task_step) )
+        validation.validate_algorithm( OpenTox::SubTask.create(task, i * task_step, ( i + 1 ) * task_step) )
         raise "validation '"+validation.validation_uri+"' for crossvaldation could not be finished" unless 
           validation.finished
         i += 1
@@ -395,14 +394,17 @@ module Validation
     private
     # copies datasets from an older crossvalidation on the same dataset and the same folds
     # returns true if successfull, false otherwise
-    def copy_cv_datasets( prediction_feature )
+    def copy_cv_datasets( )
+      # for downwards compatibilty: search prediction_feature=nil is ok
       cvs = Crossvalidation.find( { 
         :dataset_uri => self.dataset_uri, 
         :num_folds => self.num_folds, 
         :stratified => self.stratified, 
         :random_seed => self.random_seed,
         :loo => self.loo,
-        :finished => true} ).reject{ |cv| cv.id == self.id }
+        :finished => true} ).reject{ |cv| (cv.id == self.id || 
+                                          (cv.prediction_feature && 
+                                           cv.prediction_feature != self.prediction_feature)) }
       cvs.each do |cv|
         next if AA_SERVER and !OpenTox::Authorization.authorized?(cv.crossvalidation_uri,"GET",self.subjectid)
         tmp_val = []
@@ -420,7 +422,8 @@ module Validation
                        :crossvalidation_id => self.id,
                        :crossvalidation_fold => v.crossvalidation_fold,
                        :prediction_feature => prediction_feature,
-                       :algorithm_uri => self.algorithm_uri }
+                       :algorithm_uri => self.algorithm_uri,
+                       :algorithm_params => self.algorithm_params }
         end
         if tmp_val.size == self.num_folds.to_i
           @tmp_validations = tmp_val
@@ -433,111 +436,78 @@ module Validation
     
     # creates cv folds (training and testdatasets)
     # stores uris in validation objects 
-    def create_new_cv_datasets( prediction_feature, task = nil )
+    def create_new_cv_datasets( task = nil )
       LOGGER.debug "creating datasets for crossvalidation"
       orig_dataset = Lib::DatasetCache.find(self.dataset_uri,self.subjectid)
       raise OpenTox::NotFoundError.new "Dataset not found: "+self.dataset_uri.to_s unless orig_dataset
       
-      if self.loo=="true"
-        shuffled_compounds = orig_dataset.compounds
-      else
-        shuffled_compounds = orig_dataset.compounds.shuffle( self.random_seed )
-      end
+      train_dataset_uris = []
+      test_dataset_uris = []
       
-      unless self.stratified    
+      meta = { DC.creator => self.crossvalidation_uri }
+      case stratified
+      when "false"
+        if self.loo=="true"
+          shuffled_compounds = orig_dataset.compounds
+        else
+          shuffled_compounds = orig_dataset.compounds.shuffle( self.random_seed )
+        end  
         split_compounds = shuffled_compounds.chunk( self.num_folds.to_i )
-      else
-        class_compounds = {} # "inactive" => compounds[], "active" => compounds[] .. 
-        accept_values = orig_dataset.accept_values(prediction_feature)
-        raise OpenTox::BadRequestError.new("cannot apply stratification (not implemented for regression), acceptValue missing for prediction-feature '"+
-          prediction_feature.to_s+"' in dataset '"+dataset_uri.to_s+"'") unless accept_values and accept_values.size>0
-        accept_values.each do |value|
-          class_compounds[value] = []
-          shuffled_compounds.each do |c|
-            #PENDING accept values are type string, data_entries may be boolean
-            class_compounds[value] << c if orig_dataset.data_entries[c][prediction_feature].collect{|v| v.to_s}.include?(value)
-          end
-        end
-        LOGGER.debug "stratified cv: different class values: "+class_compounds.keys.join(", ")
-        LOGGER.debug "stratified cv: num instances for each class value: "+class_compounds.values.collect{|c| c.size}.join(", ")
-      
-        split_class_compounds = [] # inactive_compounds[fold_i][], active_compounds[fold_i][], ..
-        class_compounds.values.each do |compounds|
-          split_class_compounds << compounds.chunk( self.num_folds.to_i )
-        end
-        LOGGER.debug "stratified cv: splits for class values: "+split_class_compounds.collect{ |c| c.collect{ |cc| cc.size }.join("/") }.join(", ")
-        
-        # we cannot just merge the splits of the different class_values of each fold
-        # this could lead to folds, which sizes differ for more than 1 compound
-        split_compounds = []
-        split_class_compounds.each do |split_comp|
-          # step 1: sort current split in ascending order
-          split_comp.sort!{|x,y| x.size <=> y.size }
-          # step 2: add splits
-          (0..self.num_folds.to_i-1).each do |i|
-            unless split_compounds[i]
-              split_compounds[i] = split_comp[i]
+        LOGGER.debug "cv: num instances for each fold: "+split_compounds.collect{|c| c.size}.join(", ")
+          
+        self.num_folds.to_i.times do |n|
+          test_compounds = []
+          train_compounds = []
+          self.num_folds.to_i.times do |nn|
+            compounds = split_compounds[nn]
+            if n == nn
+              compounds.each{ |compound| test_compounds << compound}
             else
-              split_compounds[i] += split_comp[i]
-            end
+              compounds.each{ |compound| train_compounds << compound}
+            end 
           end
-          # step 3: sort (total) split in descending order
-          split_compounds.sort!{|x,y| y.size <=> x.size }
+          raise "internal error, num test compounds not correct,"+
+            " is '#{test_compounds.size}', should be '#{(shuffled_compounds.size/self.num_folds.to_i)}'" unless 
+            (shuffled_compounds.size/self.num_folds.to_i - test_compounds.size).abs <= 1 
+          raise "internal error, num train compounds not correct, should be '"+(shuffled_compounds.size-test_compounds.size).to_s+
+            "', is '"+train_compounds.size.to_s+"'" unless shuffled_compounds.size - test_compounds.size == train_compounds.size
+          datasetname = 'dataset fold '+(n+1).to_s+' of '+self.num_folds.to_s        
+          meta[DC.title] = "training "+datasetname 
+          LOGGER.debug "training set: "+datasetname+"_train, compounds: "+train_compounds.size.to_s
+          train_dataset_uri = orig_dataset.split( train_compounds, orig_dataset.features.keys, 
+            meta, self.subjectid ).uri
+          train_dataset_uris << train_dataset_uri
+          meta[DC.title] = "test "+datasetname
+          LOGGER.debug "test set:     "+datasetname+"_test, compounds: "+test_compounds.size.to_s
+          test_features = orig_dataset.features.keys.dclone - [self.prediction_feature]
+          test_dataset_uri = orig_dataset.split( test_compounds, test_features, 
+            meta, self.subjectid ).uri
+          test_dataset_uris << test_dataset_uri
+        end
+      when /true|super/
+        if stratified=="true"
+          features = [ self.prediction_feature ] 
+        else
+          features = nil
         end
+        train_datasets, test_datasets = stratified_k_fold_split(orig_dataset,meta,
+          "NA",self.num_folds.to_i,@subjectid,self.random_seed, features)
+        train_dataset_uris = test_datasets.collect{|d| d.uri}
+        test_dataset_uris = test_datasets.collect{|d| d.uri}
+      else
+        raise OpenTox::BadRequestError.new
       end
-      LOGGER.debug "cv: num instances for each fold: "+split_compounds.collect{|c| c.size}.join(", ")
-      
-      test_features = orig_dataset.features.keys.dclone - [prediction_feature]
       
       @tmp_validations = []
-      
-      (1..self.num_folds.to_i).each do |n|
-        
-        datasetname = 'cv'+self.id.to_s +
-               #'_d'+orig_dataset.name.to_s +
-               '_f'+n.to_s+'of'+self.num_folds.to_s+
-               '_r'+self.random_seed.to_s+
-               '_s'+self.stratified.to_s 
-        source = self.crossvalidation_uri
-        
-        test_compounds = []
-        train_compounds = []
-        
-        (1..self.num_folds.to_i).each do |nn|
-          compounds = split_compounds.at(nn-1)
-          
-          if n == nn
-            compounds.each{ |compound| test_compounds.push(compound)}
-          else
-            compounds.each{ |compound| train_compounds.push(compound)}
-          end 
-        end
-        
-        raise "internal error, num test compounds not correct" unless (shuffled_compounds.size/self.num_folds.to_i - test_compounds.size).abs <= 1 
-        raise "internal error, num train compounds not correct, should be '"+(shuffled_compounds.size-test_compounds.size).to_s+
-          "', is '"+train_compounds.size.to_s+"'" unless shuffled_compounds.size - test_compounds.size == train_compounds.size
-        
-        LOGGER.debug "training set: "+datasetname+"_train, compounds: "+train_compounds.size.to_s
-        #train_dataset_uri = orig_dataset.create_new_dataset( train_compounds, orig_dataset.features, datasetname + '_train', source ) 
-        train_dataset_uri = orig_dataset.split( train_compounds, orig_dataset.features.keys, 
-          { DC.title => datasetname + '_train', DC.creator => source }, self.subjectid ).uri
-        
-        LOGGER.debug "test set:     "+datasetname+"_test, compounds: "+test_compounds.size.to_s
-        #test_dataset_uri = orig_dataset.create_new_dataset( test_compounds, test_features, datasetname + '_test', source )
-        test_dataset_uri = orig_dataset.split( test_compounds, test_features, 
-          { DC.title => datasetname + '_test', DC.creator => source }, self.subjectid ).uri
-        
-        #make sure self.id is set
-        #self.save if self.new?
+      self.num_folds.to_i.times do |n|
         tmp_validation = { :validation_type => "crossvalidation",
-                           :training_dataset_uri => train_dataset_uri, 
-                           :test_dataset_uri => test_dataset_uri,
+                           :training_dataset_uri => train_dataset_uris[n], 
+                           :test_dataset_uri => test_dataset_uris[n],
                            :test_target_dataset_uri => self.dataset_uri,
-                           :crossvalidation_id => self.id, :crossvalidation_fold => n,
-                           :prediction_feature => prediction_feature,
+                           :crossvalidation_id => self.id, :crossvalidation_fold => (n+1),
+                           :prediction_feature => self.prediction_feature,
                            :algorithm_uri => self.algorithm_uri }
         @tmp_validations << tmp_validation
-        
         task.progress( n / self.num_folds.to_f * 100 ) if task
       end
     end
@@ -636,7 +606,7 @@ module Validation
     
     # splits a dataset into test and training dataset
     # returns map with training_dataset_uri and test_dataset_uri
-    def self.train_test_dataset_split( orig_dataset_uri, prediction_feature, subjectid, stratified=false, split_ratio=nil, random_seed=nil, task=nil )
+    def self.train_test_dataset_split( orig_dataset_uri, prediction_feature, subjectid, stratified="false", split_ratio=nil, random_seed=nil, task=nil )
       split_ratio=0.67 unless split_ratio
       split_ratio = split_ratio.to_f
       random_seed=1 unless random_seed
@@ -652,15 +622,25 @@ module Validation
           "' not found in dataset, features are: \n"+
           orig_dataset.features.keys.inspect unless orig_dataset.features.include?(prediction_feature)
       else
-        LOGGER.warn "no prediciton feature given, all features included in test dataset"
+        LOGGER.warn "no prediciton feature given, all features will be included in test dataset"
       end
       
-      if stratified
+      meta = { DC.creator => $url_provider.url_for('/training_test_split',:full) }
+      
+      case stratified
+      when /true|super/
+        if stratified=="true"
+          raise OpenTox::BadRequestError.new "prediction feature required for stratified splits" unless prediction_feature
+          features = [prediction_feature]
+        else
+          LOGGER.warn "prediction feature is ignored for super-stratified splits" if prediction_feature
+          features = nil
+        end
         r_util = OpenTox::RUtil.new 
-        split_sets = r_util.stratified_split( orig_dataset, "NA", df, split_ratio, random_seed )
+        train, test = r_util.stratified_split( orig_dataset, meta, "NA", split_ratio, @subjectid, random_seed, features )
         r_util.quit_r
-        result = {:training_dataset_uri => split_sets[0], :test_dataset_uri => split_sets[1]}
-      else
+        result = {:training_dataset_uri => train.uri, :test_dataset_uri => test.uri}
+      when "false"
         compounds = orig_dataset.compounds
         raise OpenTox::BadRequestError.new "Cannot split datset, num compounds in dataset < 2 ("+compounds.size.to_s+")" if compounds.size<2
         split = (compounds.size*split_ratio).to_i
@@ -674,22 +654,18 @@ module Validation
         test_compounds = compounds[(split+1)..-1]
         task.progress(33) if task
   
+        meta[DC.title] = "Training dataset split of "+orig_dataset.uri
         result = {}
         result[:training_dataset_uri] = orig_dataset.split( training_compounds,
-          orig_dataset.features.keys, 
-          { DC.title => "Training dataset split of "+orig_dataset.title.to_s, 
-            DC.creator => $url_provider.url_for('/training_test_split',:full) },
-          subjectid ).uri
+          orig_dataset.features.keys, meta, subjectid ).uri
         task.progress(66) if task
   
+        meta[DC.title] = "Test dataset split of "+orig_dataset.uri
         result[:test_dataset_uri] = orig_dataset.split( test_compounds,
-          orig_dataset.features.keys.dclone - [prediction_feature], 
-          { DC.title => "Test dataset split of "+orig_dataset.title.to_s, 
-            DC.creator => $url_provider.url_for('/training_test_split',:full) },
-          subjectid ).uri
+          orig_dataset.features.keys.dclone - [prediction_feature], meta, subjectid ).uri
         task.progress(100) if task  
         
-        if !stratified and ENV['RACK_ENV'] =~ /test|debug/
+        if ENV['RACK_ENV'] =~ /test|debug/
           raise OpenTox::NotFoundError.new "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless 
             Lib::DatasetCache.find(result[:training_dataset_uri],subjectid)
           test_data = Lib::DatasetCache.find result[:test_dataset_uri],subjectid
@@ -698,8 +674,9 @@ module Validation
           raise "Test dataset num coumpounds != "+(compounds.size-split-1).to_s+", instead: "+
             test_data.compounds.size.to_s+"\n"+test_data.to_yaml unless test_data.compounds.size==(compounds.size-1-split)
         end
-        
         LOGGER.debug "split done, training dataset: '"+result[:training_dataset_uri].to_s+"', test dataset: '"+result[:test_dataset_uri].to_s+"'"
+      else
+        raise OpenTox::BadRequestError.new "stratified != false|true|super, is #{stratified}"
       end
       result
     end