adjust to new wrapper, replace activerecord with datamapper (remove activerecord code), is_classification to feature_type

33 files changed, 959 insertions, 502 deletions

diff --git a/db/migrate/000_drop_validations.rb b/db/migrate/000_drop_validations.rb
index fb70967..fb70967 100644..100755
--- a/db/migrate/000_drop_validations.rb
+++ b/db/migrate/000_drop_validations.rb
diff --git a/db/migrate/001_init_validation.rb b/db/migrate/001_init_validation.rb
index 2189568..2189568 100644..100755
--- a/db/migrate/001_init_validation.rb
+++ b/db/migrate/001_init_validation.rb
diff --git a/db/migrate/002_init_reports.rb b/db/migrate/002_init_reports.rb
index 8029223..dcf44a1 100644..100755
--- a/db/migrate/002_init_reports.rb
+++ b/db/migrate/002_init_reports.rb
@@ -22,9 +22,9 @@ class InitReports < ActiveRecord::Migration
 
   def self.down
     drop_table :report_datum if table_exists? :report_datum
-    if @@config[:reports] and @@config[:reports][:report_dir]
+    if CONFIG[:reports] and CONFIG[:reports][:report_dir]
       ["validation", "crossvalidation", "algorithm_comparison"].each do |t|
-        dir = File.join(@@config[:reports][:report_dir],t)
+        dir = File.join(CONFIG[:reports][:report_dir],t)
         if File.exist?(dir)
           puts "deleting dir "+dir.to_s
           FileUtils.rm_rf(dir)
diff --git a/docbook-xsl-1.76.1/.CatalogManager.properties.example b/docbook-xsl-1.76.1/.CatalogManager.properties.example
index 23434e1..23434e1 100644..100755
--- a/docbook-xsl-1.76.1/.CatalogManager.properties.example
+++ b/docbook-xsl-1.76.1/.CatalogManager.properties.example
diff --git a/docbook-xsl-1.76.1/.urilist b/docbook-xsl-1.76.1/.urilist
index 2751d8d..2751d8d 100644..100755
--- a/docbook-xsl-1.76.1/.urilist
+++ b/docbook-xsl-1.76.1/.urilist
diff --git a/example.rb b/example.rb
index 3562dd0..1b4ee98 100644..100755
--- a/example.rb
+++ b/example.rb
@@ -5,15 +5,16 @@ class Example
   
   @@file=File.new("data/hamster_carcinogenicity.yaml","r")
   @@file_type="text/x-yaml"
-  @@model=File.join @@config[:services]["opentox-model"],"1"
-  @@feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)")
-  @@predicted_feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)_lazar_classification")
-  @@alg = File.join @@config[:services]["opentox-algorithm"],"lazar"
-  @@alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
-  @@data=File.join @@config[:services]["opentox-dataset"],"1"
-  @@train_data=File.join @@config[:services]["opentox-dataset"],"2"
-  @@test_data=File.join @@config[:services]["opentox-dataset"],"3"
-  @@prediction_data=File.join @@config[:services]["opentox-dataset"],"5"
+  @@model=File.join CONFIG[:services]["opentox-model"],"1"
+  #@@feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)")
+  @@feature= File.join CONFIG[:services]["opentox-dataset"],"1/feature/hamster_carcinogenicity"
+  #@@predicted_feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)_lazar_classification")
+  @@alg = File.join CONFIG[:services]["opentox-algorithm"],"lazar"
+  @@alg_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
+  @@data=File.join CONFIG[:services]["opentox-dataset"],"1"
+  @@train_data=File.join CONFIG[:services]["opentox-dataset"],"2"
+  @@test_data=File.join CONFIG[:services]["opentox-dataset"],"3"
+  @@prediction_data=File.join CONFIG[:services]["opentox-dataset"],"5"
   @@css_file="http://apps.ideaconsult.net:8080/ToxPredict/style/global.css"
   
   @@summary=""
@@ -28,7 +29,7 @@ class Example
     end
     file.close
     
-    sub = { "validation_service" => @@config[:services]["opentox-validation"].chomp("/"), 
+    sub = { "validation_service" => CONFIG[:services]["opentox-validation"].chomp("/"), 
             "validation_id" => "1",
             "model_uri" => @@model,
             "dataset_uri" => @@data,
@@ -42,7 +43,7 @@ class Example
             "crossvalidation_report_id" => "2",
             "css_file" => @@css_file,
             "prediction_dataset_uri" => @@prediction_data,
-            "predicted_feature" => @@predicted_feature,
+            #"predicted_feature" => @@predicted_feature,
             "qmrf_id" => "1"}
     
     sub.each do |k,v|
@@ -53,31 +54,39 @@ class Example
   
   # creates the resources that are requested by the examples
   def self.prepare_example_resources
-    OpenTox::Task.as_task("prepare examples", "n/a") do |task|
+    task = OpenTox::Task.create("prepare examples", "n/a") do |task|
       @@summary = ""
       #delete validations
       log "delete validations"
-      ActiveRecord::Base.logger = Logger.new("/dev/null")
-      ActiveRecord::Migrator.migrate('db/migrate', 0 )
-      ActiveRecord::Migrator.migrate('db/migrate', 1 )
-      ActiveRecord::Migrator.migrate('db/migrate', 2 )
+      Lib::Validation.auto_migrate!
+      Lib::Crossvalidation.auto_migrate!
+      #ActiveRecord::Base.logger = Logger.new("/dev/null")
+      #ActiveRecord::Migrator.migrate('db/migrate', 0 )
+      #ActiveRecord::Migrator.migrate('db/migrate', 1 )
+      #ActiveRecord::Migrator.migrate('db/migrate', 2 )
       
       #delete all qmrf reports
       ReachReports::QmrfReport.auto_migrate!
       
-      #delete_all(@@config[:services]["opentox-dataset"])
-      log OpenTox::RestClientWrapper.delete @@config[:services]["opentox-dataset"]
+      #delete_all(CONFIG[:services]["opentox-dataset"])
+      log OpenTox::RestClientWrapper.delete CONFIG[:services]["opentox-dataset"]
       task.progress(10)
       
       log "upload dataset"
       halt 400,"File not found: "+@@file.path.to_s unless File.exist?(@@file.path)
+      #data = File.read(@@file.path)
+      #data_uri = OpenTox::RestClientWrapper.post(CONFIG[:services]["opentox-dataset"],{:content_type => @@file_type},data).chomp("\n")
       data = File.read(@@file.path)
-      data_uri = OpenTox::RestClientWrapper.post(@@config[:services]["opentox-dataset"],{:content_type => @@file_type},data).chomp("\n")
+      dataset = OpenTox::Dataset.create
+      dataset.load_yaml(data)
+      dataset.save
+      data_uri = dataset.uri
+      log "-> "+data_uri
       task.progress(20)
       
       log "train-test-validation"
-      #delete_all(@@config[:services]["opentox-model"])
-      OpenTox::RestClientWrapper.delete @@config[:services]["opentox-model"]
+      #delete_all(CONFIG[:services]["opentox-model"])
+      OpenTox::RestClientWrapper.delete CONFIG[:services]["opentox-model"]
       
       split_params = Validation::Util.train_test_dataset_split(data_uri, URI.decode(@@feature), 0.9, 1)
       v = Validation::Validation.new :training_dataset_uri => split_params[:training_dataset_uri], 
@@ -93,7 +102,7 @@ class Example
       cv.perform_cv( URI.decode(@@feature), @@alg_params, OpenTox::SubTask.new(task, 40, 70) )
       
       log "create validation report"
-      rep = Reports::ReportService.instance(File.join(@@config[:services]["opentox-validation"],"report"))
+      rep = Reports::ReportService.instance(File.join(CONFIG[:services]["opentox-validation"],"report"))
       rep.delete_all_reports("validation")
       rep.create_report("validation",v.validation_uri)
       task.progress(80)
@@ -111,6 +120,7 @@ class Example
       log "done"
       @@summary
     end
+    task.uri
   end
   
   # performs all curl calls listed in examples after ">>>", next line is added if line ends with "\"
@@ -163,7 +173,7 @@ class Example
         result.chomp!
         result.gsub!(/\n/, " \\n ")
         if ($?==0)
-          if OpenTox::Utils.task_uri?(result)
+          if result.task_uri?
             log "wait for task: "+result
             result = Lib::TestUtil.wait_for_task(result)
           end
diff --git a/lib/active_record_setup.rb b/lib/active_record_setup.rb
index 3682c7a..404bfec 100644..100755
--- a/lib/active_record_setup.rb
+++ b/lib/active_record_setup.rb
@@ -1,17 +1,17 @@
 
-gem "activerecord", "= 2.3.8"
-gem "ar-extensions", "= 0.9.2"
-['rubygems', 'logger', 'active_record', 'ar-extensions', 'opentox-ruby-api-wrapper' ].each do |g|
+#gem "activerecord", "= 2.3.8"
+#gem "ar-extensions", "= 0.9.2"
+['rubygems', 'logger', 'active_record', 'opentox-ruby' ].each do |g| #'ar-extensions',
     require g
 end
 
 unless ActiveRecord::Base.connected?
   ActiveRecord::Base.establish_connection(  
-     :adapter => @@config[:database][:adapter],
-     :host => @@config[:database][:host],
-     :database => @@config[:database][:database],
-     :username => @@config[:database][:username],
-     :password => @@config[:database][:password]
+     :adapter => CONFIG[:database][:adapter],
+     :host => CONFIG[:database][:host],
+     :database => CONFIG[:database][:database],
+     :username => CONFIG[:database][:username],
+     :password => CONFIG[:database][:password]
   )
   ActiveRecord::Base.logger = Logger.new("/dev/null")
 end
@@ -19,6 +19,9 @@ end
 class ActiveRecord::Base
   
   def self.find_like(filter_params)
+    
+    raise "find like removed"
+    
     #puts "params before "+filter_params.inspect
     filter_params.keys.each do |k|
       key = k.to_s
diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb
index 0175a0c..52683a6 100644..100755
--- a/lib/ot_predictions.rb
+++ b/lib/ot_predictions.rb
@@ -15,7 +15,7 @@ module Lib
       return @compounds[instance_index]
     end
   
-    def initialize(is_classification, test_dataset_uri, test_target_dataset_uri, 
+    def initialize(feature_type, test_dataset_uri, test_target_dataset_uri, 
       prediction_feature, prediction_dataset_uri, predicted_variable, task=nil)
       
         LOGGER.debug("loading prediciton via test-dataset:'"+test_dataset_uri.to_s+
@@ -53,38 +53,47 @@ module Lib
           raise "prediction_feature not found in test_target_dataset\n"+
                 "prediction_feature: '"+prediction_feature.to_s+"'\n"+
                 "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+
-                "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.index(prediction_feature)==nil
+                "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil
         end
         
+        test_dataset.load_all
         @compounds = test_dataset.compounds
         LOGGER.debug "test dataset size: "+@compounds.size.to_s
         raise "test dataset is empty" unless @compounds.size>0
-        class_values = is_classification ? OpenTox::Feature.domain(prediction_feature) : nil
+        class_values = feature_type=="classification" ? OpenTox::Feature.new(prediction_feature).domain : nil
         
         actual_values = []
         @compounds.each do |c|
-          value = test_target_dataset.get_value(c, prediction_feature)
-          
-          if is_classification
-            value = value.to_s unless value==nil
-            raise "illegal class_value of actual value "+value.to_s+" class: "+
-              value.class.to_s unless value==nil or class_values.index(value)!=nil
-            actual_values.push class_values.index(value) 
-          else
-            begin
-              value = value.to_f unless value==nil or value.is_a?(Numeric)
-            rescue
-              LOGGER.warn "no numeric value for regression: '"+value.to_s+"'"
-              value = nil
-            end
-            actual_values.push value
+          case feature_type
+          when "classification"
+            actual_values << classification_value(test_target_dataset, c, prediction_feature, class_values)
+          when "regression"
+            actual_values << regression_value(test_target_dataset, c, prediction_feature)
           end
         end
         task.progress(40) if task # loaded actual values
         
         prediction_dataset = OpenTox::Dataset.find prediction_dataset_uri
         raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset
-        raise "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+prediction_dataset.features.inspect if prediction_dataset.features.index(predicted_variable)==nil
+        
+        # TODO: remove LAZAR_PREDICTION_DATASET_HACK
+        no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil
+        if no_prediction_feature
+          one_entry_per_compound = prediction_dataset.data_entries.keys.size == @compounds.size
+          @compounds.each do |c|
+            if prediction_dataset.data_entries[c].size != 1
+              one_entry_per_compound = false
+              break
+            end
+          end
+          msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+
+            prediction_dataset.features.keys.inspect
+          if one_entry_per_compound
+            LOGGER.warn msg
+          else
+            raise msg
+          end
+        end
         
         raise "more predicted than test compounds test:"+@compounds.size.to_s+" < prediction:"+
           prediction_dataset.compounds.size.to_s if @compounds.size < prediction_dataset.compounds.size
@@ -102,38 +111,79 @@ module Lib
             predicted_values << nil
             confidence_values << nil
           else
-            if is_classification
-              value = prediction_dataset.get_predicted_class(c, predicted_variable)
-              value = value.to_s unless value==nil
-              raise "illegal class_value of predicted value "+value.to_s+" class: "+value.class.to_s unless value==nil or class_values.index(value)!=nil
-              predicted_values << class_values.index(value)
-            else
-              value = prediction_dataset.get_predicted_regression(c, predicted_variable)
-              begin
-                value = value.to_f unless value==nil or value.is_a?(Numeric)
-              rescue
-                LOGGER.warn "no numeric value for regression: '"+value.to_s+"'"
-                value = nil
-              end
-              predicted_values << value
+            case feature_type
+            when "classification"
+              # TODO: remove LAZAR_PREDICTION_DATASET_HACK
+              predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, class_values)
+            when "regression"
+              predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable)
+            end
+            # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable)
+            conf = 1
+            begin
+              feature = prediction_dataset.data_entries[c].keys[0]
+              feature_data = prediction_dataset.features[feature]
+              conf = feature_data[OT.confidence] if feature_data[OT.confidence]!=nil 
+            rescue
+              LOGGER.warn "could not get confidence"
             end
-            confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable)
+            confidence_values << conf
           end
         end
         task.progress(80) if task # loaded predicted values and confidence
         
-        super(predicted_values, actual_values, confidence_values, is_classification, class_values)
+        super(predicted_values, actual_values, confidence_values, feature_type, class_values)
         raise "illegal num compounds "+num_info if  @compounds.size != @predicted_values.size
         task.progress(100) if task # done with the mathmatics
     end
     
+    private
+    def regression_value(dataset, compound, feature)
+      v = value(dataset, compound, feature)
+      begin
+        v = v.to_f unless v==nil or v.is_a?(Numeric)
+        v
+      rescue
+        LOGGER.warn "no numeric value for regression: '"+v.to_s+"'"
+        nil
+      end
+    end
+    
+    def classification_value(dataset, compound, feature, class_values)
+      v = value(dataset, compound, feature)
+      i = class_values.index(v)
+      raise "illegal class_value of predicted value "+v.to_s+" class: "+v.class.to_s unless v==nil or i!=nil
+      i
+    end
+    
+    def value(dataset, compound, feature)
+      
+      if feature==nil
+        v = dataset.data_entries[compound].values[0]
+      else
+        v = dataset.data_entries[compound][feature]
+      end
+      raise "no array" unless v.is_a?(Array)
+      if v.size>1
+        raise "multiple values"
+      elsif v.size==1
+        v = v[0]
+      else
+        v = nil
+      end
+      raise "array" if v.is_a?(Array)
+      v = nil if v.to_s.size==0
+      v
+    end
 
+    public
     def compute_stats
     
       res = {}
-      if @is_classification
+      case @feature_type
+      when "classification"
         (Lib::VAL_CLASS_PROPS).each{ |s| res[s] = send(s)}  
-      else
+      when "regression"
         (Lib::VAL_REGR_PROPS).each{ |s| res[s] = send(s) }  
       end
       return res
@@ -152,16 +202,18 @@ module Lib
           
           #PENDING!
           begin
-            a.push( "http://ambit.uni-plovdiv.bg:8080/ambit2/depict/cdk?search="+
-              URI.encode(OpenTox::Compound.new(:uri=>p.identifier(i)).smiles) ) if add_pic
+            #a.push( "http://ambit.uni-plovdiv.bg:8080/ambit2/depict/cdk?search="+
+            #  URI.encode(OpenTox::Compound.new(:uri=>p.identifier(i)).smiles) ) if add_pic
+            a << p.identifier(i)+"/image"
           rescue => ex
+            raise ex
             #a.push("Could not add pic: "+ex.message)
-            a.push(p.identifier(i))
+            #a.push(p.identifier(i))
           end
           
           a << (format ? p.actual_value(i).to_nice_s : p.actual_value(i))
           a << (format ? p.predicted_value(i).to_nice_s : p.predicted_value(i))
-          if p.classification?
+          if p.feature_type=="classification"
             if (p.predicted_value(i)!=nil and p.actual_value(i)!=nil)
               a << (p.classification_miss?(i) ? 1 : 0)
             else
@@ -180,7 +232,7 @@ module Lib
       header << "compound" if add_pic
       header << "actual value"
       header << "predicted value"
-      header << "missclassified" if predictions[0].classification?
+      header << "missclassified" if predictions[0].feature_type=="classification"
       header << "confidence value" if predictions[0].confidence_values_available?
       header << "compound-uri"
       res.insert(0, header)
diff --git a/lib/predictions.rb b/lib/predictions.rb
index 6e50e94..e73dda6 100644..100755
--- a/lib/predictions.rb
+++ b/lib/predictions.rb
@@ -22,13 +22,13 @@ module Lib
     def initialize( predicted_values, 
                     actual_values, 
                     confidence_values, 
-                    is_classification, 
+                    feature_type, 
                     class_domain=nil )
                     
       @predicted_values = predicted_values
       @actual_values = actual_values
       @confidence_values = confidence_values
-      @is_classification = is_classification
+      @feature_type = feature_type
       @class_domain = class_domain
       @num_classes = 1
       
@@ -36,6 +36,8 @@ module Lib
       #puts "actual:     "+actual_values.inspect
       #puts "confidence: "+confidence_values.inspect
       
+      raise "unknown feature_type: "+@feature_type.to_s unless 
+        @feature_type=="classification" || @feature_type=="regression"
       raise "no predictions" if @predicted_values.size == 0
       num_info = "predicted:"+@predicted_values.size.to_s+
         " confidence:"+@confidence_values.size.to_s+" actual:"+@actual_values.size.to_s
@@ -54,15 +56,16 @@ module Lib
       #  @confidence_values=nil
       #end
       
-      if @is_classification
+      case @feature_type
+      when "classification"
         raise "class_domain missing while performing classification" unless @class_domain
         @num_classes = @class_domain.size
         raise "num classes < 2" if @num_classes<2
         { "predicted"=>@predicted_values, "actual"=>@actual_values }.each do |s,values|
           values.each{ |v| raise "illegal "+s+" classification-value ("+v.to_s+"),"+
-            "has to be either nil or index of predicted-values" if v!=nil and (v<0 or v>@num_classes)}
+            "has to be either nil or index of predicted-values" if v!=nil and (!v.is_a?(Numeric) or v<0 or v>@num_classes)}
         end
-      else
+      when "regresssion"
         raise "class_domain != nil while performing regression" if @class_domain
         { "predicted"=>@predicted_values, "actual"=>@actual_values }.each do |s,values|
           values.each{ |v| raise "illegal "+s+" regression-value ("+v.to_s+"),"+
@@ -83,7 +86,8 @@ module Lib
       @num_predicted = 0
       @num_unpredicted = 0
       
-      if @is_classification
+      case @feature_type
+      when "classification"
         @confusion_matrix = []
         @class_domain.each do |v|
           @confusion_matrix.push( Array.new( @num_classes, 0 ) )
@@ -91,7 +95,7 @@ module Lib
         
         @num_correct = 0
         @num_incorrect = 0
-      else
+      when "regression"
         @sum_error = 0
         @sum_abs_error = 0
         @sum_squared_error = 0
@@ -122,14 +126,15 @@ module Lib
         else
           @num_predicted += 1
           
-          if @is_classification
+          case @feature_type
+          when "classification"
             @confusion_matrix[actual_value][predicted_value] += 1
             if (predicted_value == actual_value)
               @num_correct += 1
             else
               @num_incorrect += 1
             end
-          else
+          when "regression"
             delta = predicted_value - actual_value
             @sum_error += delta
             @sum_abs_error += delta.abs
@@ -156,13 +161,13 @@ module Lib
     end
     
     def percent_correct
-      raise "no classification" unless @is_classification
+      raise "no classification" unless @feature_type=="classification"
       return 0 if @num_with_actual_value==0
       return 100 * @num_correct / @num_with_actual_value.to_f
     end
     
     def percent_incorrect
-      raise "no classification" unless @is_classification
+      raise "no classification" unless @feature_type=="classification"
       return 0 if @num_with_actual_value==0
       return 100 * @num_incorrect / @num_with_actual_value.to_f
     end
@@ -190,17 +195,17 @@ module Lib
     end
 
     def num_correct
-      raise "no classification" unless @is_classification
+      raise "no classification" unless @feature_type=="classification"
       return @num_correct
     end
 
     def num_incorrect
-      raise "no classification" unless @is_classification
+      raise "no classification" unless @feature_type=="classification"
       return @num_incorrect
     end
     
     def num_unclassified
-      raise "no classification" unless @is_classification
+      raise "no classification" unless @feature_type=="classification"
       return @num_unpredicted
     end
     
@@ -209,7 +214,7 @@ module Lib
     #     and values: <int-value>
     def confusion_matrix
       
-      raise "no classification" unless @is_classification
+      raise "no classification" unless @feature_type=="classification"
       res = {}
       (0..@num_classes-1).each do |actual|
           (0..@num_classes-1).each do |predicted|
@@ -505,9 +510,10 @@ module Lib
     end
   
     def predicted_value(instance_index)
-      if @is_classification
+      case @feature_type 
+      when "classification"
         @predicted_values[instance_index]==nil ? nil : @class_domain[@predicted_values[instance_index]]
-      else
+      when "regression"
         @predicted_values[instance_index]
       end
     end
@@ -517,9 +523,10 @@ module Lib
     end
     
     def actual_value(instance_index)
-      if @is_classification
+      case @feature_type 
+      when "classification"
         @actual_values[instance_index]==nil ? nil : @class_domain[@actual_values[instance_index]]
-      else
+      when "regression"
         @actual_values[instance_index]
       end
     end
@@ -529,13 +536,13 @@ module Lib
     end      
     
     def classification_miss?(instance_index)
-      raise "no classification" unless @is_classification
+      raise "no classification" unless @feature_type=="classification"
       return false if predicted_value(instance_index)==nil or actual_value(instance_index)==nil
       return predicted_value(instance_index) != actual_value(instance_index)
     end
     
-    def classification?
-      @is_classification
+    def feature_type
+      @feature_type
     end
     
     def confidence_values_available?
diff --git a/lib/test_util.rb b/lib/test_util.rb
index ecab76c..0deee04 100644..100755
--- a/lib/test_util.rb
+++ b/lib/test_util.rb
@@ -10,11 +10,11 @@ module Lib
     end
     
     def self.wait_for_task(uri)
-      if OpenTox::Utils.task_uri?(uri)
+      if uri.task_uri?
         task = OpenTox::Task.find(uri)
         task.wait_for_completion
         raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error?
-        uri = task.resultURI
+        uri = task.result_uri
       end
       return uri
     end
diff --git a/lib/validation_db.rb b/lib/validation_db.rb
index cb3ece7..83b7e2f 100644..100755
--- a/lib/validation_db.rb
+++ b/lib/validation_db.rb
@@ -49,11 +49,40 @@ module Lib
   VAL_MERGE_AVG = VAL_PROPS_AVG + VAL_CLASS_PROPS_SINGLE_AVG + VAL_CLASS_PROPS_PER_CLASS_AVG + VAL_REGR_PROPS
   
 
-  class Validation < ActiveRecord::Base
-    serialize :classification_statistics
-    serialize :regression_statistics
+#  class Validation < ActiveRecord::Base
+#    serialize :classification_statistics
+#    serialize :regression_statistics
+#    
+#    alias_attribute :date, :created_at
     
-    alias_attribute :date, :created_at
+  class Validation 
+    include DataMapper::Resource
+  
+    property :id, Serial
+    property :validation_type, String, :length => 255
+    property :model_uri, String, :length => 255
+    property :algorithm_uri, String, :length => 255
+    property :training_dataset_uri, String, :length => 255
+    property :test_target_dataset_uri, String, :length => 255
+    property :test_dataset_uri, String, :length => 255
+    property :prediction_dataset_uri, String, :length => 255
+    property :prediction_feature, String, :length => 255
+    property :created_at, DateTime
+    property :num_instances, Integer
+    property :num_without_class, Integer
+    property :num_unpredicted, Integer
+    property :crossvalidation_id, Integer
+    property :crossvalidation_fold, Integer
+    property :real_runtime, Float
+    property :percent_without_class, Float
+    property :percent_unpredicted, Float
+    property :classification_statistics, Object
+    property :regression_statistics, Object
+    property :finished, Boolean, :default => false
+    
+    def date
+      created_at
+    end
     
     def validation_uri
       $sinatra.url_for("/"+self.id.to_s, :full)
@@ -77,8 +106,23 @@ module Lib
     
   end
   
-  class Crossvalidation < ActiveRecord::Base
-    alias_attribute :date, :created_at
+#  class Crossvalidation < ActiveRecord::Base
+#    alias_attribute :date, :created_at
+  class Crossvalidation 
+    include DataMapper::Resource
+  
+    property :id, Serial
+    property :algorithm_uri, String, :length => 255
+    property :dataset_uri, String, :length => 255
+    property :created_at, DateTime
+    property :num_folds, Integer, :default => 10
+    property :random_seed, Integer, :default => 1
+    property :finished, Boolean, :default => false
+    property :stratified, Boolean, :default => false
+    
+    def date
+      created_at
+    end
     
     def crossvalidation_uri
       $sinatra.url_for("/crossvalidation/"+self.id.to_s, :full) if self.id
@@ -88,7 +132,8 @@ module Lib
     # in terms of dataset_uri,num_folds,stratified,random_seed
     # further conditions can be specified in __conditions__
     def self.find_all_uniq(conditions={})
-      cvs = Lib::Crossvalidation.find(:all, :conditions => conditions)
+      #cvs = Lib::Crossvalidation.find(:all, :conditions => conditions)
+      cvs = Lib::Crossvalidation.all(:conditions => conditions)
       uniq = []
       cvs.each do |cv|
         match = false
@@ -105,3 +150,9 @@ module Lib
     end
   end
 end
+
+
+Lib::Validation.auto_upgrade!
+Lib::Validation.raise_on_save_failure = true
+Lib::Crossvalidation.auto_upgrade!
+Lib::Crossvalidation.raise_on_save_failure = true
\ No newline at end of file
diff --git a/nightly/nightly.rb b/nightly/nightly.rb
index eb802a3..550d14c 100644..100755
--- a/nightly/nightly.rb
+++ b/nightly/nightly.rb
@@ -21,7 +21,7 @@ class Nightly
     validationExamples = ValidationExamples.select(select)
     return "please \"select\" validation examples:\n"+ValidationExamples.list if validationExamples.size==0
     
-    task_uri = OpenTox::Task.as_task("Build nightly","nightly-validation-test-service",{:select => select, :dry_run => dry_run}) do |task|
+    task = OpenTox::Task.create("Build nightly","nightly-validation-test-service") do |task| #,{:select => select, :dry_run => dry_run})
       LOGGER.info("Building nightly report")
       
       benchmarks = validationExamples.collect{ |e| ValidationBenchmark.new(e) }
@@ -98,12 +98,12 @@ class Nightly
       end
       
       #benchmarks.collect{|b| b.uris}.join(",")
-      File.join(@@config[:services]["opentox-validation"],"nightly")
+      File.join(CONFIG[:services]["opentox-validation"],"nightly")
     end
     if defined?(halt)
-      halt 202,task_uri+"\n"
+      halt 202,task.uri+"\n"
     else
-      return task_uri+"\n"
+      return task.uri+"\n"
     end
   end
   
diff --git a/reach_reports/reach_application.rb b/reach_reports/reach_application.rb
index d82bc11..4187c1b 100644..100755
--- a/reach_reports/reach_application.rb
+++ b/reach_reports/reach_application.rb
@@ -1,5 +1,5 @@
 
-[ 'rubygems', 'sinatra', 'sinatra/url_for', 'opentox-ruby-api-wrapper' ].each do |lib|
+[ 'rubygems', 'sinatra', 'sinatra/url_for', 'opentox-ruby' ].each do |lib|
   require lib
 end
 
@@ -67,7 +67,7 @@ post '/reach_report/:type' do
   #puts "creating "+type+" report "+params.inspect
   result_uri = ReachReports.create_report(type,params,request.env["rack.input"])
   
-  if OpenTox::Utils.task_uri?(result_uri)
+  if result_uri and result_uri.task_uri?
     halt 202,result_uri+"\n"   
   else
     result_uri+"\n"
diff --git a/reach_reports/reach_persistance.rb b/reach_reports/reach_persistance.rb
index 2cbcd71..a36e05f 100644..100755
--- a/reach_reports/reach_persistance.rb
+++ b/reach_reports/reach_persistance.rb
@@ -1188,41 +1188,11 @@ module ReachReports
 #    end
 #  end
 
-  QsarSoftware.auto_upgrade!  
-  QsarIdentifier.auto_upgrade!
-  
-  QmrfAuthor.auto_upgrade!
-  ModelAuthor.auto_upgrade!
-  Reference.auto_upgrade!
-  QsarGeneralInformation.auto_upgrade!
-  
-  ModelEndpoint.auto_upgrade!
-  QsarEndpoint.auto_upgrade!
-  
-  AlgorithmExplicit.auto_upgrade!
-  AlgorithmsDescriptor.auto_upgrade!
-  DescriptorsGenerationSoftware.auto_upgrade!
-  QsarAlgorithm.auto_upgrade!
-  
-  AppDomainSoftware.auto_upgrade!
-  QsarApplicabilityDomain.auto_upgrade!
-  
-  TrainingSetData.auto_upgrade!
-  QsarRobustness.auto_upgrade!
-  
-  ValidationSetData.auto_upgrade!
-  QsarPredictivity.auto_upgrade!
-  
-  QsarInterpretation.auto_upgrade!
-  
-  Bibliography.auto_upgrade!
-  AttachmentTrainingData.auto_upgrade!
-  AttachmentValidationData.auto_upgrade!
-  AttachmentDocument.auto_upgrade!
-  QsarMiscellaneous.auto_upgrade!
-  
-  QmrfSummary.auto_upgrade!
-  
-  QmrfReport.auto_upgrade!
-  
+  [ QsarSoftware, QsarIdentifier, QmrfAuthor, ModelAuthor, Reference, QsarGeneralInformation, ModelEndpoint, QsarEndpoint, AlgorithmExplicit, 
+    AlgorithmsDescriptor, DescriptorsGenerationSoftware, QsarAlgorithm, AppDomainSoftware, QsarApplicabilityDomain, TrainingSetData, 
+    QsarRobustness, ValidationSetData, QsarPredictivity, QsarInterpretation, Bibliography, AttachmentTrainingData, AttachmentValidationData, 
+    AttachmentDocument, QsarMiscellaneous, QmrfSummary, QmrfReport ].each do |model|
+    model.auto_upgrade!
+    model.raise_on_save_failure = true
+  end
 end
\ No newline at end of file
diff --git a/reach_reports/reach_service.rb b/reach_reports/reach_service.rb
index f297f26..808d08a 100644..100755
--- a/reach_reports/reach_service.rb
+++ b/reach_reports/reach_service.rb
@@ -26,13 +26,14 @@ module ReachReports
     case type
     when /(?i)QMRF/
       if params[:model_uri]
-        result_uri = OpenTox::Task.as_task( "Create "+type+" report", 
-          $sinatra.url_for("/reach_report/"+type, :full), params ) do |task|
+        task = OpenTox::Task.create( "Create "+type+" report", 
+          $sinatra.url_for("/reach_report/"+type, :full) ) do |task| #, params
             
           report = ReachReports::QmrfReport.new :model_uri => params[:model_uri]
           build_qmrf_report(report, task)
           report.report_uri
         end
+        result_uri = task.uri
       elsif xml_data and (input = xml_data.read).to_s.size>0
         report = ReachReports::QmrfReport.new
         ReachReports::QmrfReport.from_xml(report,input)
@@ -61,17 +62,18 @@ module ReachReports
   def self.build_qmrf_report(r, task=nil)
     
     #puts r.model_uri
-    model = OpenTox::Model::PredictionModel.find(r.model_uri)
-    classification = model.classification?
+    model = OpenTox::Model::Generic.find(r.model_uri)
+    raise "model not found "+r.model_uri.to_s unless model
+    feature_type = model.feature_type
     
     # chapter 1
     r.qsar_identifier = QsarIdentifier.new
-    r.qsar_identifier.qsar_title = model.title
+    r.qsar_identifier.qsar_title = model.metadata[DC.title]
     # TODO QSAR_models -> sparql same endpoint     
     r.qsar_identifier.qsar_software << QsarSoftware.new( :url => model.uri, 
-      :name => model.title, :contact => model.creator )
-    algorithm = OpenTox::Algorithm::Generic.find(model.algorithm) if model.algorithm
-    r.qsar_identifier.qsar_software << QsarSoftware.new( :url => algorithm.uri, :name => algorithm.title )
+      :name => model.metadata[DC.title], :contact => model.metadata[DC.creator] )
+    algorithm = OpenTox::Algorithm::Generic.find(model.metadata[OT.algorithm]) if model.metadata[OT.algorithm]
+    r.qsar_identifier.qsar_software << QsarSoftware.new( :url => algorithm.uri, :name => algorithm.metadata[DC.title] )
     task.progress(10) if task
 
     #chpater 2
@@ -79,7 +81,7 @@ module ReachReports
     r.qsar_general_information.qmrf_date = DateTime.now.to_s
     # EMPTY: qmrf_authors, qmrf_date_revision, qmrf_revision
     # TODO: model_authors ?
-    r.qsar_general_information.model_date = model.date.to_s
+    r.qsar_general_information.model_date = model.metadata[DC.date].to_s
     # TODO: references?
     # EMPTY: info_availablity
     # TODO: related_models = find qmrf reports for QSAR_models 
@@ -88,11 +90,11 @@ module ReachReports
     # chapter 3
     # TODO "model_species" ?
     r.qsar_endpoint = QsarEndpoint.new
-    model.predictedVariables.each do |p|
+    model.metadata[OT.predictedVariables].each do |p|
       r.qsar_endpoint.model_endpoint << ModelEndpoint.new( :name => p )
-    end
+    end if model.metadata[OT.predictedVariables]
     # TODO "endpoint_comments" => "3.3", "endpoint_units" => "3.4",
-    r.qsar_endpoint.endpoint_variable =  model.dependentVariables if model.dependentVariables
+    r.qsar_endpoint.endpoint_variable =  model.metadata[OT.dependentVariables] if model.metadata[OT.dependentVariables]
     # TODO "endpoint_protocol" => "3.6", "endpoint_data_quality" => "3.7",
     task.progress(30) if task
     
@@ -108,9 +110,9 @@ module ReachReports
 
     #training_dataset = model.trainingDataset ? OpenTox::Dataset.find(model.trainingDataset+"/metadata") : nil
     begin
-      training_dataset = model.trainingDataset ? OpenTox::Dataset.find(model.trainingDataset) : nil
+      training_dataset = model.metadata[OT.trainingDataset] ? OpenTox::Dataset.find(model.metadata[OT.trainingDataset]) : nil
     rescue
-      LOGGER.warn "build qmrf: training_dataset not found "+model.trainingDataset.to_s
+      LOGGER.warn "build qmrf: training_dataset not found "+model.metadata[OT.trainingDataset].to_s
     end
     task.progress(50) if task
 
@@ -129,27 +131,27 @@ module ReachReports
     
     val_datasets = []
     
-    if model.algorithm
-      cvs = Lib::Crossvalidation.find_all_uniq({:algorithm_uri => model.algorithm})
+    if algorithm
+      cvs = Lib::Crossvalidation.find_all_uniq({:algorithm_uri => algorithm.uri})
       # PENDING: cv classification/regression hack
       cvs = cvs.delete_if do |cv|
         val = Validation::Validation.first( :all, :conditions => { :crossvalidation_id => cv.id } )
-        (val.classification_statistics!=nil) != classification
+        (val.classification_statistics!=nil) != (feature_type=="classification")
       end
       
-      lmo = [ "found "+cvs.size.to_s+" crossvalidation/s for algorithm '"+model.algorithm+"'" ]
+      lmo = [ "found "+cvs.size.to_s+" crossvalidation/s for algorithm '"+algorithm.uri.to_s+"'" ]
       if cvs.size>0
         cvs_same_data = []
         cvs_other_data = []
         cvs.each do |cv|
-          if cv.dataset_uri == model.trainingDataset
+          if cv.dataset_uri == model.metadata[OT.trainingDataset]
             cvs_same_data << cv
           else
             cvs_other_data << cv
           end
         end
         lmo << cvs_same_data.size.to_s+" crossvalidations/s where performed on the training dataset of the model ("+
-          model.trainingDataset.to_s+")"
+          model.metadata[OT.trainingDataset].to_s+")"
         lmo << cvs_other_data.size.to_s+" crossvalidations/s where performed on the other datasets"        
         lmo << ""
         
@@ -162,15 +164,16 @@ module ReachReports
             val_datasets << cv.dataset_uri
             lmo << "settings: num-folds="+cv.num_folds.to_s+", random-seed="+cv.random_seed.to_s+", stratified:"+cv.stratified.to_s
             val  = YAML.load( OpenTox::RestClientWrapper.get File.join(cv.crossvalidation_uri,"statistics") )
-            if classification
+            case feature_type
+            when "classification"
               lmo << "percent_correct: "+val[:classification_statistics][:percent_correct].to_s
               lmo << "weighted AUC: "+val[:classification_statistics][:weighted_area_under_roc].to_s
-            else
+            when "regression"
               lmo << "root_mean_squared_error: "+val[:regression_statistics][:root_mean_squared_error].to_s
               lmo << "r_square "+val[:regression_statistics][:r_square].to_s
             end
             reports = OpenTox::RestClientWrapper.get File.join(CONFIG[:services]["opentox-validation"],"report/crossvalidation?crossvalidation_uris="+cv.crossvalidation_uri)
-            if reports and reports.size>0
+            if reports and reports.chomp.size>0
               lmo << "for more info see report: "+reports.split("\n")[0]
             else
               lmo << "for more info see report: not yet created for '"+cv.crossvalidation_uri+"'"
@@ -187,7 +190,8 @@ module ReachReports
     # "lmo" => "6.9", "yscrambling" => "6.10", "bootstrap" => "6.11", "other_statistics" => "6.12",
 
     LOGGER.debug "looking for validations with "+{:model_uri => model.uri}.inspect
-    vals = Lib::Validation.find(:all, :conditions => {:model_uri => model.uri})
+    #vals = Lib::Validation.find(:all, :conditions => {:model_uri => model.uri})
+    vals = Lib::Validation.all({:model_uri => model.uri})
     uniq_vals = []
     vals.each do |val|
       match = false
@@ -212,10 +216,11 @@ module ReachReports
         v << "validation: "+validation.validation_uri
         v << "dataset (see 9.3 Validation data): "+validation.test_dataset_uri
         val_datasets << validation.test_dataset_uri
-        if classification
+        case feature_type
+        when "classification"
           v << "percent_correct: "+validation.classification_statistics[:percent_correct].to_s
           v << "weighted AUC: "+validation.classification_statistics[:weighted_area_under_roc].to_s
-        else
+        when "regression"
           v << "root_mean_squared_error: "+validation.regression_statistics[:root_mean_squared_error].to_s
           v << "r_square "+validation.regression_statistics[:r_square].to_s
         end
@@ -252,7 +257,7 @@ module ReachReports
     r.qsar_miscellaneous.attachment_training_data << AttachmentTrainingData.new( 
       { :description => training_dataset.title, 
         :filetype => "owl-dl", 
-        :url => model.trainingDataset} ) if training_dataset
+        :url => training_dataset.uri} ) if training_dataset
         
     val_datasets.each do |data_uri|
       begin
diff --git a/reach_reports/reach_test.rb b/reach_reports/reach_test.rb
index 4a18f6d..13625e5 100644..100755
--- a/reach_reports/reach_test.rb
+++ b/reach_reports/reach_test.rb
@@ -7,7 +7,7 @@ require 'rack/test'
 require 'lib/test_util.rb'
 require 'test/test_examples.rb'
 
-LOGGER = MyLogger.new(STDOUT)
+LOGGER = OTLogger.new(STDOUT)
 LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S "
 LOGGER.formatter = Logger::Formatter.new
 
@@ -115,7 +115,7 @@ class ReachTest < Test::Unit::TestCase
 #    puts last_response.body
     
     #model_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/model/173393"
-    model_uri = "http://localhost/model/6"
+    model_uri = "http://localhost/model/1"
     #http://localhost/majority/class/model/15
     #model_uri = "http://localhost/majority/class/model/15"
    # model_uri = "http://localhost/majority/class/model/91"
@@ -141,15 +141,15 @@ class ReachTest < Test::Unit::TestCase
 #    puts "RDF"
 #    puts last_response.body
 
-    get '/reach_report/qmrf/'+id,nil,'HTTP_ACCEPT' => "application/qmrf-xml"
-    puts "XML"
-    puts last_response.body
+    #get '/reach_report/qmrf/'+id,nil,'HTTP_ACCEPT' => "application/qmrf-xml"
+    #puts "XML"
+    #puts last_response.body
     
     
     #r = ReachReports::QmrfReport.find_like( :QSAR_title => "Hamster")
     #puts r.collect{|rr| "report with id:"+rr.id.to_s}.inspect
     
-    File.new("/home/martin/tmp/qmr_rep_del_me.xml","w").puts last_response.body
+    #File.new("/home/martin/tmp/qmr_rep_del_me.xml","w").puts last_response.body
     #File.new("/home/martin/win/home/qmr_rep_del_me.xml","w").puts last_response.body
     #File.new("/home/martin/info_home/.public_html/qmr_rep_del_me.xml","w").puts last_response.body
   end
diff --git a/report/environment.rb b/report/environment.rb
index 1f62d0e..12e3272 100644..100755
--- a/report/environment.rb
+++ b/report/environment.rb
@@ -1,18 +1,19 @@
 
 ['rubygems', 'logger', 'fileutils', 'sinatra', 'sinatra/url_for', 'rest_client', 
   'yaml', 'fileutils', 'mime/types', 'abbrev', 'rinruby',
-  'rexml/document',  'ruby-plot', 'opentox-ruby-api-wrapper' ].each do |g|
+  'rexml/document',  'ruby-plot', 'opentox-ruby' ].each do |g|
     require g
 end
 gem 'ruby-plot', '= 0.0.2'
 
-R.quit
+#R.quit
 
 module Reports
 end
 
 require "lib/ot_predictions.rb"
-require "lib/active_record_setup.rb"
+#require "lib/active_record_setup.rb"
+require "lib/data_mapper_util.rb"
 
 require "report/plot_factory.rb"
 require "report/xml_report.rb"
diff --git a/report/report_application.rb b/report/report_application.rb
index a61a2df..6d6353f 100644..100755
--- a/report/report_application.rb
+++ b/report/report_application.rb
@@ -123,11 +123,11 @@ delete '/report/:type/:id' do
 end
 
 post '/report/:type' do
-  task_uri = OpenTox::Task.as_task("Create report",url_for("/report/"+params[:type], :full), params) do |task|
+  task = OpenTox::Task.create("Create report",url_for("/report/"+params[:type], :full)) do |task| #,params
     perform do |rs|
       rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil,task)
     end
   end
   content_type "text/uri-list"
-  halt 202,task_uri+"\n"
+  halt 202,task.uri+"\n"
 end
diff --git a/report/report_content.rb b/report/report_content.rb
index eeb65e9..1345e6f 100644..100755
--- a/report/report_content.rb
+++ b/report/report_content.rb
@@ -192,7 +192,7 @@ class Reports::ReportContent
       class_domain = validation_set.get_class_domain
       class_domain.size.times do |i|
         class_value = class_domain[i]
-        image_title = image_titles ? image_titles[i] : "ROC Plot for class-value '"+class_value+"'"
+        image_title = image_titles ? image_titles[i] : "ROC Plot for class-value '"+class_value.to_s+"'"
         image_caption = image_captions ? image_captions[i] : nil
         plot_file_name = "roc_plot"+@tmp_file_count.to_s+".svg"
         @tmp_file_count += 1
@@ -201,7 +201,7 @@ class Reports::ReportContent
           Reports::PlotFactory.create_roc_plot( plot_file_path, prediction_set, class_value, split_set_attribute, false )#prediction_set.size>1 )
           @xml_report.add_imagefigure(section_roc, image_title, plot_file_name, "SVG", 120, image_caption)
         rescue RuntimeError => ex
-          msg = "WARNING could not create roc plot for class value '"+class_value+"': "+ex.message
+          msg = "WARNING could not create roc plot for class value '"+class_value.to_s+"': "+ex.message
           LOGGER.error(msg)
           rm_tmp_file(plot_file_name)
           @xml_report.add_paragraph(section_roc, msg)
@@ -245,7 +245,7 @@ class Reports::ReportContent
         image_title = image_titles[i]
       else
         if class_value!=nil
-          image_title = rank_attribute.to_s+" Ranking Plot for class-value '"+class_value+"'"
+          image_title = rank_attribute.to_s+" Ranking Plot for class-value '"+class_value.to_s+"'"
         else 
           image_title = rank_attribute.to_s+" Ranking Plot"
         end
diff --git a/report/report_factory.rb b/report/report_factory.rb
index ee66df7..11e9bfa 100644..100755
--- a/report/report_factory.rb
+++ b/report/report_factory.rb
@@ -65,11 +65,12 @@ module Reports::ReportFactory
 
     report = Reports::ReportContent.new("Validation report")
     
-    if (val.classification?)
+    case val.feature_type
+    when "classification"
       report.add_result(validation_set, [:validation_uri] + VAL_ATTR_TRAIN_TEST + VAL_ATTR_CLASS, "Results", "Results")
       report.add_roc_plot(validation_set)
       report.add_confusion_matrix(val)
-    else #regression
+    when "regression"
       report.add_result(validation_set, [:validation_uri] + VAL_ATTR_TRAIN_TEST + VAL_ATTR_REGR, "Results", "Results")
       report.add_regression_plot(validation_set, :model_uri)
     end
@@ -90,8 +91,8 @@ module Reports::ReportFactory
     raise Reports::BadRequest.new("num validations ("+validation_set.size.to_s+") is not equal to num folds ("+
       validation_set.unique_value(:num_folds).to_s+")") unless validation_set.unique_value(:num_folds)==validation_set.size
     raise Reports::BadRequest.new("num different folds is not equal to num validations") unless validation_set.num_different_values(:crossvalidation_fold)==validation_set.size
-    raise Reports::BadRequest.new("validations must be either all regression, "+
-      +"or all classification validations") unless validation_set.all_classification? or validation_set.all_regression?  
+    raise Reports::BadRequest.new("validations must have unique feature type, i.e. must be either all regression, "+
+      +"or all classification validations") unless validation_set.unique_feature_type  
     pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) )
     
     merged = validation_set.merge([:crossvalidation_id])
@@ -100,14 +101,15 @@ module Reports::ReportFactory
     #puts merged.get_values(:percent_correct_variance, false).inspect
     report = Reports::ReportContent.new("Crossvalidation report")
     
-    if (validation_set.all_classification?)
+    case validation_set.unique_feature_type
+    when "classification"
       report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold],"Mean Results","Mean Results")
       report.add_roc_plot(validation_set, nil, "ROC Plots over all folds")
       report.add_roc_plot(validation_set, :crossvalidation_fold)
       report.add_confusion_matrix(merged.validations[0])
       report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_CLASS-[:num_folds],
         "Results","Results",nil,"validation")
-    else #regression
+    when "regression"
       report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold],"Mean Results","Mean Results")
       report.add_regression_plot(validation_set, :crossvalidation_fold)
       report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_REGR-[:num_folds], "Results","Results")
@@ -124,8 +126,8 @@ module Reports::ReportFactory
     
     #validation_set.to_array([:test_dataset_uri, :model_uri, :algorithm_uri], false).each{|a| puts a.inspect}
     raise Reports::BadRequest.new("num validations is not >1") unless validation_set.size>1
-    raise Reports::BadRequest.new("validations must be either all regression, "+
-      "or all classification validations") unless validation_set.all_classification? or validation_set.all_regression?
+    raise Reports::BadRequest.new("validations must have unique feature type, i.e. must be either all regression, "+
+      +"or all classification validations") unless validation_set.unique_feature_type
     raise Reports::BadRequest.new("number of different algorithms <2: "+
       validation_set.get_values(:algorithm_uri).inspect) if validation_set.num_different_values(:algorithm_uri)<2
       
@@ -168,7 +170,8 @@ module Reports::ReportFactory
       
     end
 
-    if (validation_set.all_classification?)
+    case validation_set.unique_feature_type
+    when "classification"
       attributes = VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold]
       attributes = ([ :dataset_uri ] + attributes).uniq
       
@@ -190,7 +193,7 @@ module Reports::ReportFactory
         report.end_section
       end
       
-    else # regression
+    when "regression"
       raise Reports::BadRequest.new("algorithm comparison for regression not yet implemented")
     end
     task.progress(100) if task
diff --git a/report/report_persistance.rb b/report/report_persistance.rb
index 1bd37b0..d24c792 100644..100755
--- a/report/report_persistance.rb
+++ b/report/report_persistance.rb
@@ -181,14 +181,28 @@ end
 
 module Reports
   
-  class ReportData < ActiveRecord::Base
+  #class ReportData < ActiveRecord::Base
+#    serialize :validation_uris
+#    serialize :crossvalidation_uris
+#    serialize :algorithm_uris
+#    serialize :model_uris
+#    alias_attribute :date, :created_at
+
+  class ReportData 
+    include DataMapper::Resource 
   
-    serialize :validation_uris
-    serialize :crossvalidation_uris
-    serialize :algorithm_uris
-    serialize :model_uris
+    property :id, Serial
+    property :report_uri, String, :length => 255
+    property :report_type, String, :length => 255
+    property :created_at, DateTime
+    property :validation_uris, Object 
+    property :crossvalidation_uris, Object
+    property :model_uris, Object
+    property :algorithm_uris, Object
     
-    alias_attribute :date, :created_at
+    def date
+      created_at
+    end
     
     def get_content_as_hash
       map = {}
@@ -216,23 +230,40 @@ module Reports
       raise "report meta data missing" unless meta_data
       report = ReportData.new(meta_data)
       report.save #to set id
-      report.attributes = { :report_type => type, :report_uri => uri_provider.get_uri(type, report.id) }
-      report.save
+#      report.attributes = { :report_type => type, :report_uri => uri_provider.get_uri(type, report.id) }
+#      report.save
+      report.update :report_type => type, :report_uri => uri_provider.get_uri(type, report.id)
       new_report_with_id(report_content, type, report.id)
     end
     
     def list_reports(type, filter_params={})
       filter_params["report_type"]=type unless filter_params.has_key?("report_type")
-      ReportData.find_like(filter_params).delete_if{|r| r.report_type!=type}.collect{ |r| r.id }
+      #ReportData.find_like(filter_params).delete_if{|r| r.report_type!=type}.collect{ |r| r.id }
+      
+      filter_params = Lib::DataMapperUtil.check_params(ReportData, filter_params)
+      # unfortunately, datamapper does not allow searching in Objects
+      # do filtering for list = Object params manually
+      list_params = {}
+      [:validation_uris, :crossvalidation_uris, :algorithm_uris, :model_uris].each do |l|
+        list_params[l] = filter_params.delete(l) if filter_params.has_key?(l)
+      end
+      
+      reports = ReportData.all(filter_params).delete_if{|r| r.report_type!=type}
+      list_params.each do |k,v|
+        reports = reports.delete_if{ |r| !r.send(k).include?(v) }
+      end
+      reports.collect{ |r| r.id }
     end
     
     def get_report(type, id, format, force_formating, params)
       
-      begin
-        report = ReportData.find(:first, :conditions => {:id => id, :report_type => type})
-      rescue ActiveRecord::RecordNotFound
-        raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.")
-      end
+      report = ReportData.first({:id => id, :report_type => type})
+      raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") unless report
+#      begin
+#        report = ReportData.find(:first, :conditions => {:id => id, :report_type => type})
+#      rescue ActiveRecord::RecordNotFound
+#        raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.")
+#      end
   
       case format
       when "application/rdf+xml"
@@ -245,13 +276,51 @@ module Reports
     end
     
     def delete_report(type, id)
-      begin
-        report = ReportData.find(:first, :conditions => {:id => id, :report_type => type})
-      rescue ActiveRecord::RecordNotFound
-        raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.")
-      end
-      ReportData.delete(id)
+#      begin
+#        report = ReportData.find(:first, :conditions => {:id => id, :report_type => type})
+#      rescue ActiveRecord::RecordNotFound
+#        raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.")
+#      end
+#      ReportData.delete(id)
+      report = ReportData.first({:id => id, :report_type => type})
+      raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") unless report
+      report.destroy
       super      
     end
   end
 end
+
+Reports::ReportData.auto_upgrade!
+Reports::ReportData.raise_on_save_failure = true
+
+#module Reports
+#  def self.check_filter_params(model, filter_params)
+#    prop_names = model.properties.collect{|p| p.name.to_s}
+#    filter_params.keys.each do |k|
+#      key = k.to_s
+#      unless prop_names.include?(key)
+#        key = key.from_rdf_format
+#        unless prop_names.include?(key)
+#          key = key+"_uri"
+#          unless prop_names.include?(key)
+#            key = key+"s"
+#            unless prop_names.include?(key)
+#              err = "no attribute found: '"+k.to_s+"'"
+#              if $sinatra
+#                $sinatra.halt 400,err
+#              else
+#                raise err
+#              end
+#            end
+#          end
+#        end
+#      end
+#      filter_params[key] = filter_params.delete(k)
+#    end
+#    filter_params
+#  end
+#  
+#  def ReportData.all( params )
+#    super Reports.check_filter_params( ReportData, params )
+#  end
+#end
diff --git a/report/report_test.rb b/report/report_test.rb
index 5351c7d..f579802 100644..100755
--- a/report/report_test.rb
+++ b/report/report_test.rb
@@ -33,7 +33,10 @@ class Reports::ApplicationTest < Test::Unit::TestCase
     #puts uri
     #get uri
     
-    get '/report/validation/117',nil,'HTTP_ACCEPT' => "text/html"     
+    get '/report/crossvalidation',:model=>"http://localhost/majority/class/model/101"
+    puts last_response.body.to_s
+     
+    #get '/report/validation/117',nil,'HTTP_ACCEPT' => "text/html"     
     #post '/report/validation/1/format_html',:css_style_sheet=>"http://apps.ideaconsult.net:8180/ToxPredict/style/global.css"
     
     #post 'http://ot.validation.de/report/validation',:validation_uris=>"http://ot.validation.de/1"
@@ -119,12 +122,12 @@ end
 #class Reports::ReportServiceTest < Test::Unit::TestCase
 #  include Lib::TestUtil
 #
-#  WS_VAL = @@config[:services]["opentox-validation"]
-#  WS_DATA=@@config[:services]["opentox-dataset"]
+#  WS_VAL = CONFIG[:services]["opentox-validation"]
+#  WS_DATA=CONFIG[:services]["opentox-dataset"]
 #  FILE=File.new("data/hamster_carcinogenicity.owl","r")
 #  
-#  WS_CLASS_ALG=File.join(@@config[:services]["opentox-algorithm"],"lazar")
-#  WS_FEATURE_ALG=File.join(@@config[:services]["opentox-algorithm"],"fminer")
+#  WS_CLASS_ALG=File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+#  WS_FEATURE_ALG=File.join(CONFIG[:services]["opentox-algorithm"],"fminer")
 #  
 #  #WS_CLASS_ALG_2="localhost:4008/algorithm"
 #  #WS_FEATURE_ALG_2=nil
diff --git a/report/validation_access.rb b/report/validation_access.rb
index a0b6355..170cdfd 100644..100755
--- a/report/validation_access.rb
+++ b/report/validation_access.rb
@@ -36,8 +36,8 @@ class Reports::ValidationAccess
     raise "not implemented"
   end
   
-  # is validation classification?
-  def classification?(validation)
+  # is validation classification/regression?
+  def feature_type(validation)
     raise "not implemented"
   end
   
@@ -49,9 +49,9 @@ end
 
 class Reports::ValidationDB < Reports::ValidationAccess
   
-  def initialize()
-    @model_store = {}
-  end
+#  def initialize()
+#    @model_store = {}
+#  end
   
   def resolve_cv_uris(validation_uris)
     res = []
@@ -60,13 +60,15 @@ class Reports::ValidationDB < Reports::ValidationAccess
         cv_id = u.split("/")[-1].to_i
         cv = nil
         begin
-          cv = Lib::Crossvalidation.find( cv_id )
+          #cv = Lib::Crossvalidation.find( cv_id )
+          cv = Lib::Crossvalidation.get( cv_id )
         rescue => ex
           raise "could not access crossvalidation with id "+validation_id.to_s+", error-msg: "+ex.message
         end
         raise Reports::BadRequest.new("crossvalidation with id '"+cv_id.to_s+"' not found") unless cv
         raise Reports::BadRequest.new("crossvalidation with id '"+cv_id.to_s+"' not finished") unless cv.finished
-        res += Lib::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s}
+        #res += Lib::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s}
+        res += Lib::Validation.all( :crossvalidation_id => cv_id ).collect{|v| v.validation_uri.to_s }
       else
         res += [u.to_s]
       end
@@ -82,7 +84,8 @@ class Reports::ValidationDB < Reports::ValidationAccess
       (validation_id.to_i > 0 || validation_id.to_s=="0" )
     v = nil
     begin
-      v = Lib::Validation.find(validation_id)
+      #v = Lib::Validation.find(validation_id)
+      v = Lib::Validation.get(validation_id)
     rescue => ex
       raise "could not access validation with id "+validation_id.to_s+", error-msg: "+ex.message
     end
@@ -102,7 +105,8 @@ class Reports::ValidationDB < Reports::ValidationAccess
     
   def init_cv(validation)
     
-    cv = Lib::Crossvalidation.find(validation.crossvalidation_id)
+    #cv = Lib::Crossvalidation.find(validation.crossvalidation_id)
+    cv = Lib::Crossvalidation.get(validation.crossvalidation_id)
     raise Reports::BadRequest.new "no crossvalidation found with id "+validation.crossvalidation_id.to_s unless cv
     
     Lib::CROSS_VAL_PROPS.each do |p|
@@ -111,35 +115,39 @@ class Reports::ValidationDB < Reports::ValidationAccess
   end
 
   def get_predictions(validation, task=nil)
-    Lib::OTPredictions.new( validation.classification?, validation.test_dataset_uri, 
+    Lib::OTPredictions.new( validation.feature_type, validation.test_dataset_uri, 
       validation.test_target_dataset_uri, validation.prediction_feature, validation.prediction_dataset_uri, 
       validation.predicted_variable, task)
   end
   
   def get_class_domain( validation )
-    OpenTox::Feature.domain( validation.prediction_feature )
+    OpenTox::Feature.new( validation.prediction_feature ).domain
   end
   
-  def classification?( validation )
-    get_model(validation).classification?
+  def feature_type( validation )
+    OpenTox::Model::Generic.new(validation.model_uri).feature_type
+    #get_model(validation).classification?
   end
   
   def predicted_variable(validation)
-    get_model(validation).predictedVariables
-  end
-  
-  private
-  def get_model(validation)
     raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation)
-    model = @model_store[validation.model_uri]
-    unless model
-      model = OpenTox::Model::PredictionModel.find(validation.model_uri)
-      raise "model not found '"+validation.model_uri+"'" unless validation.model_uri && model
-      @model_store[validation.model_uri] = model
-    end
-    return model
+    model = OpenTox::Model::Generic.find(validation.model_uri)
+    raise Reports::NotFound.new "model not found '"+validation.model_uri+"'" unless model
+    model.metadata[OT.predictedVariables]
+    #get_model(validation).predictedVariables
   end
   
+#  private
+#  def get_model(validation)
+#    raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation)
+#    model = @model_store[validation.model_uri]
+#    unless model
+#      model = OpenTox::Model::PredictionModel.find(validation.model_uri)
+#      raise "model not found '"+validation.model_uri+"'" unless validation.model_uri && model
+#      @model_store[validation.model_uri] = model
+#    end
+#    return model
+#  end
   
 end
 
diff --git a/report/validation_data.rb b/report/validation_data.rb
index d558c88..c4eed12 100644..100755
--- a/report/validation_data.rb
+++ b/report/validation_data.rb
@@ -107,11 +107,11 @@ module Reports
       @class_domain
     end
     
-    # is classification validation? cache to save resr-calls
+    # is classification/regression validation? cache to save rest-calls
     #
-    def classification?
-      return @is_classification if @is_classification!=nil
-      @is_classification = @@validation_access.classification?(self) 
+    def feature_type
+      return @feature_type if @feature_type!=nil
+      @feature_type = @@validation_access.feature_type(self) 
     end
     
     def predicted_variable
@@ -130,7 +130,7 @@ module Reports
     def validation_report_uri
       #puts "searching for validation report: "+self.validation_uri.to_s
       return @validation_report_uri if @validation_report_uri!=nil
-      ids = @@persistance.list_reports("validation",{:validation=>validation_uri })
+      ids = @@persistance.list_reports("validation",{:validation_uris=>validation_uri })
       @validation_report_uri = Reports::ReportService.instance.get_uri("validation",ids[-1]) if ids and ids.size>0
     end
     
@@ -259,19 +259,12 @@ module Reports
       end
     end
     
-    # checks weather all validations are classification validations
+    # checks weather all validations are classification/regression validations
     #
-    def all_classification?
-      return unique_value("classification?")
+    def unique_feature_type
+      return unique_value("feature_type")
     end
 
-    # checks weather all validations are regression validations
-    #
-    def all_regression?
-      # WARNING, NOT TRUE: !all_classification == all_regression? 
-      return unique_value("classification?")==false
-    end
-    
     # returns a new set with all validation that have values as specified in the map
     #
     # call-seq:
@@ -348,39 +341,42 @@ module Reports
       array.push(attributes.collect{|a| a.to_s.nice_attr})
       attribute_not_nil = Array.new(attributes.size)
       @validations.each do |v|
-        index = 0
+        index = -1
         array.push(attributes.collect do |a|
+          index += 1
           if VAL_ATTR_VARIANCE.index(a)
             variance = v.send( (a.to_s+"_variance").to_sym )
           end
           
           #variance = " +- "+variance.to_nice_s if variance
-          attribute_not_nil[index] = true if remove_nil_attributes and v.send(a)!=nil
-          index += 1
           val = v.send(a)
-          
-          class_domain = get_domain_for_attr(a)
-          # get domain for classification attribute, i.e. ["true","false"]
-          if class_domain.size==1 && class_domain[0]!=nil
-            # or the attribute has a complementary value, i.e. true_positive_rate
-            # -> domain is reduced to one class value
-            raise "illegal state" unless (val.is_a?(Hash))
-            val = val[class_domain[0]]
-          end
-          
-          if variance
-            if (val.is_a?(Array))
-              raise "not implemented"
-            elsif (val.is_a?(Hash))
-              val.collect{ |i,j| i.to_nice_s+": "+j.to_nice_s + " +- " +
-                variance[i].to_nice_s  }.join(", ")
+          if val==nil || val.to_s.chomp.size==0
+            ''
+          else
+            attribute_not_nil[index] = true if remove_nil_attributes
+            
+            class_domain = get_domain_for_attr(a)
+            # get domain for classification attribute, i.e. ["true","false"]
+            if class_domain.size==1 && class_domain[0]!=nil
+              # or the attribute has a complementary value, i.e. true_positive_rate
+              # -> domain is reduced to one class value
+              raise "illegal state, value for "+a.to_s+" is no hash: '"+val.to_s+"'" unless (val.is_a?(Hash))
+              val = val[class_domain[0]]
+            end
+            
+            if variance
+              if (val.is_a?(Array))
+                raise "not implemented"
+              elsif (val.is_a?(Hash))
+                val.collect{ |i,j| i.to_nice_s+": "+j.to_nice_s + " +- " +
+                  variance[i].to_nice_s  }.join(", ")
+              else
+                val.to_nice_s + " +- " + variance.to_nice_s
+              end
             else
-              val.to_nice_s + " +- " + variance.to_nice_s
+              val.to_nice_s
             end
-          else
-            val.to_nice_s
           end
-          
         end)
       end
 
diff --git a/report/xml_report.rb b/report/xml_report.rb
index 76844b3..46f2a67 100644..100755
--- a/report/xml_report.rb
+++ b/report/xml_report.rb
@@ -184,7 +184,7 @@ module Reports
         row = Element.new("row")
         r.each do |v|
           entry = Element.new("entry")
-          if auto_link_urls && v.to_s =~ /depict/ #PENDING
+          if auto_link_urls && v.to_s =~ /depict/ || v.to_s =~ /image$/ #PENDING 
             add_image(entry, v.to_s)
           elsif auto_link_urls && v.to_s =~ /^http:\/\//
            add_url(entry, v.to_s, v.to_s)
@@ -241,7 +241,7 @@ module Reports
         end
       end
       
-      @doc.write(out,2, true, true)
+      @doc.write(out) #,2, true, true)
       out.flush
     end
   
diff --git a/report/xml_report_util.rb b/report/xml_report_util.rb
index fcb7d96..fd1a179 100644..100755
--- a/report/xml_report_util.rb
+++ b/report/xml_report_util.rb
@@ -35,21 +35,22 @@ module Reports
       end
       
       confusion = []
-      confusion.push( [ "", "", "actual" ] + [""] * num_classes )
-      confusion.push( [ "", "" ] + class_values +  [ "total"])
+      confusion.push( [ " ", " ", "actual" ] + [" "] * num_classes )
+      confusion.push( [ " ", " " ] + class_values +  [ "total"])
       
       class_values.each do |predicted|
-        row =  [ (confusion.size==2 ? "predicted" : ""), predicted ]
+        row =  [ (confusion.size==2 ? "predicted" : " "), predicted ]
         class_values.each do |actual|
           row.push( confusion_matrix[{:confusion_matrix_actual => actual, :confusion_matrix_predicted => predicted}].to_nice_s )   
         end
         row.push( sum_predicted[predicted].to_nice_s )
         confusion.push( row )  
       end
-      last_row = [ "", "total" ] 
+      last_row = [ " ", "total" ] 
       class_values.each do |actual|
           last_row.push( sum_actual[actual].to_nice_s )  
       end
+      last_row.push(" ")
       confusion.push( last_row )
       
       return confusion
diff --git a/test/test_application.rb b/test/test_application.rb
index 48cced7..c6c7963 100644..100755
--- a/test/test_application.rb
+++ b/test/test_application.rb
@@ -2,7 +2,7 @@
 post '/test_validation/?' do
   validationExamples = ValidationExamples.select(params[:select])
   return "please \"select\" a single validation example:\n"+ValidationExamples.list if validationExamples.size!=1 or validationExamples[0].size!=1
-  OpenTox::Task.as_task("Test validation",url_for("/test_validation",:full), params) do
+  task = OpenTox::Task.create("Test validation",url_for("/test_validation",:full)) do #,params
     v = validationExamples[0][0]
     ex = v.new
     ex.upload_files
@@ -13,5 +13,6 @@ post '/test_validation/?' do
     raise ex.report_error if ex.report_error
     ex.validation_uri + (params[:report] ? ","+ex.report_uri : "")
   end
+  task.uri
 end
 
diff --git a/test/test_examples.rb b/test/test_examples.rb
index 8927fb1..63fb505 100644..100755
--- a/test/test_examples.rb
+++ b/test/test_examples.rb
@@ -19,15 +19,15 @@ module ValidationExamples
   
   class MajorityIrisCrossvalidation < IrisCrossvalidation
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm")
       super
     end
   end
 
   class LazarIrisCrossvalidation < IrisCrossvalidation
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
-      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer")
       super
     end
   end
@@ -43,15 +43,65 @@ module ValidationExamples
   
   class LazarIrisSplit < IrisSplit
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
-      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer")
       super
     end
   end
   
   class MajorityIrisSplit < IrisSplit
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm")
+      super
+    end
+  end
+  
+  ########################################################################################################  
+  
+  class EPAFHMSplit < SplitTestValidation
+    def initialize
+      @dataset_file = File.new("data/EPAFHM.csv","r")
+      #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk"
+      @split_ratio = 0.95
+    end
+  end
+    
+  class LazarEPAFHMSplit < EPAFHMSplit
+    def initialize
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
+      super
+    end
+  end
+  
+  class MajorityEPAFHMSplit < EPAFHMSplit
+    def initialize
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm")
+      super
+    end
+  end
+  
+    ########################################################################################################
+  
+    class EPAFHMCrossvalidation < CrossValidation
+    def initialize
+      @dataset_file = File.new("data/EPAFHM.csv","r")
+      #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk"
+      @num_folds = 10
+    end
+  end
+  
+  class MajorityEPAFHMCrossvalidation < EPAFHMCrossvalidation
+    def initialize
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm")
+      super
+    end
+  end
+
+  class LazarEPAFHMCrossvalidation < EPAFHMCrossvalidation
+    def initialize
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
       super
     end
   end
@@ -61,21 +111,22 @@ module ValidationExamples
   class HamsterSplit < SplitTestValidation
     def initialize
       @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
-      @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+      #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+      @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity"
     end
   end
   
   class LazarHamsterSplit < HamsterSplit
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
-      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
       super
     end
   end
   
   class MajorityHamsterSplit < HamsterSplit
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm")
       super
     end
   end
@@ -85,21 +136,22 @@ module ValidationExamples
   class HamsterBootstrapping < BootstrappingValidation
     def initialize
       @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
-      @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+      #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+      @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity"
     end
   end
   
   class LazarHamsterBootstrapping < HamsterBootstrapping
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
-      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
       super
     end
   end
   
   class MajorityHamsterBootstrapping < HamsterBootstrapping
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm")
       super
     end
   end  
@@ -111,21 +163,22 @@ module ValidationExamples
       @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
       @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r")
       @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r")
-      @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+      #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+      @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity"
     end
   end
   
   class MajorityHamsterTrainingTest < HamsterTrainingTest
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm")
       super
     end
   end
   
   class LazarHamsterTrainingTest < HamsterTrainingTest
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
-      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
       super
     end
   end
@@ -135,22 +188,23 @@ module ValidationExamples
   class HamsterCrossvalidation < CrossValidation
     def initialize
       @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
-      @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+      #@prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+      @prediction_feature = "http://localhost/dataset/1/feature/hamster_carcinogenicity"
       @num_folds = 10
     end
   end
   
   class MajorityHamsterCrossvalidation < HamsterCrossvalidation
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm")
       super
     end
   end
 
   class LazarHamsterCrossvalidation < HamsterCrossvalidation
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
-      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
       super
     end
   end
@@ -166,15 +220,15 @@ module ValidationExamples
   
   class MajorityISTHamsterCrossvalidation < ISTHamsterCrossvalidation
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm")
       super
     end
   end
   
   class LazarISTHamsterCrossvalidation < ISTHamsterCrossvalidation
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
-      @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer")
       super
     end
   end
@@ -215,7 +269,7 @@ module ValidationExamples
   
   class MajorityISTRatLiverCrossvalidation < ISTRatLiverCrossvalidation
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/class/algorithm")
       super
     end
   end
@@ -283,7 +337,7 @@ module ValidationExamples
   
   class MajorityCacoTrainingTest < CacoTrainingTest
     def initialize
-      @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm")
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm")
       super
     end
   end
@@ -331,6 +385,12 @@ module ValidationExamples
       "12" => [ LazarHamsterBootstrapping, MajorityHamsterBootstrapping ],
       "12a" => [ LazarHamsterBootstrapping ],
       "12b" => [ MajorityHamsterBootstrapping ],
+      
+      "13a" =>  [ LazarEPAFHMSplit ],
+      "13b" =>  [ MajorityEPAFHMSplit ],
+      
+      "14a" =>  [ LazarEPAFHMCrossvalidation ],
+      "14b" =>  [ MajorityEPAFHMCrossvalidation ],
     }
   
   def self.list
diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb
index cc31fe5..b9a1c99 100644..100755
--- a/test/test_examples_util.rb
+++ b/test/test_examples_util.rb
@@ -4,20 +4,40 @@ module ValidationExamples
   class Util
 
     @@dataset_uris = {}
+    @@prediction_features = {}
 
-    def self.upload_dataset(file, dataset_service=@@config[:services]["opentox-dataset"], file_type="application/x-yaml")
+    def self.upload_dataset(file, dataset_service=CONFIG[:services]["opentox-dataset"]) #, file_type="application/x-yaml")
       raise "File not found: "+file.path.to_s unless File.exist?(file.path)
       if @@dataset_uris[file.path.to_s]==nil
-        data = File.read(file.path)
-        data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp
-        @@dataset_uris[file.path.to_s] = data_uri
-        LOGGER.debug "uploaded dataset: "+data_uri
+        LOGGER.debug "uploading file: "+file.path.to_s
+        if (file.path =~ /yaml$/)
+          data = File.read(file.path)
+          #data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp
+          #@@dataset_uris[file.path.to_s] = data_uri
+          #LOGGER.debug "uploaded dataset: "+data_uri
+          d = OpenTox::Dataset.create
+          d.load_yaml(data)
+          d.save
+          @@dataset_uris[file.path.to_s] = d.uri
+        elsif (file.path =~ /csv$/)
+          d = OpenTox::Dataset.create_from_csv_file(file.path)
+          raise "num features not 1 (="+d.features.keys.size.to_s+"), what to predict??" if d.features.keys.size != 1
+          @@prediction_features[file.path.to_s] = d.features.keys[0]
+          @@dataset_uris[file.path.to_s] = d.uri
+        else
+          raise "unknown file type: "+file.path.to_s
+        end
+        LOGGER.debug "uploaded dataset: "+d.uri
       else
         LOGGER.debug "file already uploaded: "+@@dataset_uris[file.path.to_s]
       end
       return @@dataset_uris[file.path.to_s]
     end
     
+    def self.prediction_feature_for_file(file)
+      @@prediction_features[file.path.to_s]
+    end
+    
     def self.build_compare_report(validation_examples)
       
       @comp = validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri
@@ -35,12 +55,12 @@ module ValidationExamples
     end
     
     def self.validation_post(uri, params)
+      
       if $test_case
-        #puts "posting: "+uri+","+params.inspect
-        $test_case.post uri,params 
+        $test_case.post uri,params
         return wait($test_case.last_response.body)
       else
-        return OpenTox::RestClientWrapper.post(File.join(@@config[:services]["opentox-validation"],uri),params).to_s
+        return OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],uri),params).to_s
       end
     end
     
@@ -50,16 +70,27 @@ module ValidationExamples
         $test_case.get uri,nil,'HTTP_ACCEPT' => accept_header 
         return wait($test_case.last_response.body)
       else
-        return OpenTox::RestClientWrapper.get(File.join(@@config[:services]["opentox-validation"],uri),{:accept => accept_header})
+        return OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"],uri),{:accept => accept_header})
+      end
+    end
+
+    def self.validation_delete(uri, accept_header='application/rdf+xml')
+      
+      if $test_case
+        $test_case.delete uri,nil,'HTTP_ACCEPT' => accept_header 
+        return wait($test_case.last_response.body)
+      else
+        return OpenTox::RestClientWrapper.delete(File.join(CONFIG[:services]["opentox-validation"],uri),{:accept => accept_header})
       end
     end
     
+    
     def self.wait(uri)
-      if OpenTox::Utils.task_uri?(uri)
-        task = OpenTox::Task.find(uri)
+      if uri.task_uri?
+        task = OpenTox::Task.find(uri.to_s.chomp)
         task.wait_for_completion
-        raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error?
-        uri = task.resultURI
+        raise "task failed: "+uri.to_s+", error is:\n"+task.description.to_s if task.error?
+        uri = task.result_uri
       end
       uri
     end
@@ -214,6 +245,10 @@ module ValidationExamples
             send("#{uri.to_s}=".to_sym, Util.upload_dataset(send(file)))
          end
       end
+     
+      if (params.include?(:prediction_feature) and @prediction_feature==nil and @dataset_uri and @dataset_file)
+        @prediction_feature = Util.prediction_feature_for_file(@dataset_file)
+      end
     end
       
     def check_requirements
@@ -222,21 +257,42 @@ module ValidationExamples
       end
     end
     
+    def delete
+      begin
+        if @validation_uri =~ /crossvalidation/
+          cv = "crossvalidation/"
+        else
+          cv = ""
+        end
+        Util.validation_delete '/'+cv+@validation_uri.split('/')[-1] if @validation_uri
+      rescue => ex
+        puts "Could not delete validation: "+ex.message
+      end
+      begin
+        Util.validation_delete '/report/'+report_type+'/'+@report_uri.split('/')[-1] if @report_uri
+      rescue => ex
+        puts "Could not delete report:' "+@report_uri+" "+ex.message
+      end
+    end
+    
     def report
       begin
         @report_uri = Util.validation_post '/report/'+report_type,{:validation_uris => @validation_uri}
       rescue => ex
+        puts "could not create report: "+ex.message
+        raise ex
         @report_error = ex.message
       end
     end
     
     def validate
-      #begin
+      begin
         @validation_uri = Util.validation_post '/'+validation_type, get_params
-      #rescue => ex
-      #  @validation_error = ex.message
-      #  LOGGER.error ex.message
-      #end
+      rescue => ex
+        puts "could not validate: "+ex.message
+        @validation_error = ex.message
+        LOGGER.error ex.message
+      end
     end
     
     def compare_yaml_vs_rdf
@@ -308,6 +364,10 @@ module ValidationExamples
     def opt_params
       [ :algorithm_params, :test_target_dataset_uri ]
     end
+    
+    def validation_type
+      "training_test_validation"
+    end
   end
   
   class SplitTestValidation < ValidationExample
diff --git a/validation/validation_application.rb b/validation/validation_application.rb
index 2a2bcb0..9c6c60c 100644..100755
--- a/validation/validation_application.rb
+++ b/validation/validation_application.rb
@@ -1,16 +1,17 @@
 
-[ 'rubygems', 'sinatra', 'sinatra/url_for', 'opentox-ruby-api-wrapper' ].each do |lib|
+[ 'rubygems', 'sinatra', 'sinatra/url_for', 'opentox-ruby' ].each do |lib|
   require lib
 end
 
 require 'lib/merge.rb'
-require 'lib/active_record_setup.rb'
+#require 'lib/active_record_setup.rb'
 
 require 'validation/validation_service.rb'
 
 get '/crossvalidation/?' do
   LOGGER.info "list all crossvalidations"
-  uri_list = Validation::Crossvalidation.find_like(params).collect{ |cv| cv.crossvalidation_uri }.join("\n")+"\n"
+  uri_list = Validation::Crossvalidation.all.collect{ |cv| cv.crossvalidation_uri }.join("\n")+"\n"
+  #uri_list = Validation::Crossvalidation.find_like(params).collect{ |cv| cv.crossvalidation_uri }.join("\n")+"\n"
   if request.env['HTTP_ACCEPT'] =~ /text\/html/
     related_links = 
       "Single validations:      "+$sinatra.url_for("/",:full)+"\n"+
@@ -28,8 +29,7 @@ get '/crossvalidation/?' do
 end
 
 post '/crossvalidation/?' do
-  content_type "text/uri-list"
-  task_uri = OpenTox::Task.as_task( "Perform crossvalidation", url_for("/crossvalidation", :full), params ) do |task|
+  task = OpenTox::Task.create( "Perform crossvalidation", url_for("/crossvalidation", :full) ) do |task| #, params
     LOGGER.info "creating crossvalidation "+params.inspect
     halt 400, "dataset_uri missing" unless params[:dataset_uri]
     halt 400, "algorithm_uri missing" unless params[:algorithm_uri]
@@ -44,20 +44,25 @@ post '/crossvalidation/?' do
     cv.perform_cv( params[:prediction_feature], params[:algorithm_params], task )
     cv.crossvalidation_uri
   end
-  halt 202,task_uri+"\n"
+  content_type 'text/uri-list' 
+  halt 503,task.uri+"\n" if task.status == "Cancelled"
+  halt 202,task.uri
 end
 
 post '/crossvalidation/cleanup/?' do
   LOGGER.info "crossvalidation cleanup, starting..."
   content_type "text/uri-list"
   deleted = []
-  Validation::Crossvalidation.find_like(params).each do |cv|
-    num_vals = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv.id } ).size
-    if cv.num_folds != num_vals
-      LOGGER.debug "delete cv with id:"+cv.id.to_s+" num-folds should be "+cv.num_folds.to_s+", is "+num_vals.to_s
+  #Validation::Crossvalidation.find_like(params).each do |cv|
+  Validation::Crossvalidation.all( { :finished => false } ).each do |cv|
+    #num_vals = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv.id } ).size
+    #num_vals = Validation::Validation.all( :crossvalidation_id => cv.id ).size
+    #if cv.num_folds != num_vals || !cv.finished
+      LOGGER.debug "delete cv with id:"+cv.id.to_s+", finished is false"
       deleted << cv.crossvalidation_uri
-      Validation::Crossvalidation.delete(cv.id)
-    end
+      #Validation::Crossvalidation.delete(cv.id)
+      cv.delete
+    #end
   end
   LOGGER.info "crossvalidation cleanup, deleted "+deleted.size.to_s+" cvs"
   deleted.join("\n")+"\n"
@@ -73,11 +78,13 @@ end
 
 get '/crossvalidation/:id' do
   LOGGER.info "get crossvalidation with id "+params[:id].to_s
-  begin
-    crossvalidation = Validation::Crossvalidation.find(params[:id])
-  rescue ActiveRecord::RecordNotFound => ex
-    halt 404, "Crossvalidation '#{params[:id]}' not found."
-  end
+#  begin
+#    #crossvalidation = Validation::Crossvalidation.find(params[:id])
+#  rescue ActiveRecord::RecordNotFound => ex
+#    halt 404, "Crossvalidation '#{params[:id]}' not found."
+#  end
+  crossvalidation = Validation::Crossvalidation.get(params[:id])
+  halt 404,"Crossvalidation '#{params[:id]}' not found." unless crossvalidation
   
   case request.env['HTTP_ACCEPT'].to_s
   when "application/rdf+xml"
@@ -104,19 +111,24 @@ end
 
 get '/crossvalidation/:id/statistics' do
   LOGGER.info "get merged validation-result for crossvalidation with id "+params[:id].to_s
-  begin
-    crossvalidation = Validation::Crossvalidation.find(params[:id])
-  rescue ActiveRecord::RecordNotFound => ex
-    halt 404, "Crossvalidation '#{params[:id]}' not found."
-  end
+#  begin
+    #crossvalidation = Validation::Crossvalidation.find(params[:id])
+#  rescue ActiveRecord::RecordNotFound => ex
+#    halt 404, "Crossvalidation '#{params[:id]}' not found."
+#  end
+  #crossvalidation = Validation::Crossvalidation.find(params[:id])
+  crossvalidation = Validation::Crossvalidation.get(params[:id])
+  
+  halt 404,"Crossvalidation '#{params[:id]}' not found." unless crossvalidation
   halt 400,"Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished
   
   Lib::MergeObjects.register_merge_attributes( Validation::Validation,
-    Lib::VAL_MERGE_AVG,Lib::VAL_MERGE_SUM,Lib::VAL_MERGE_GENERAL-[:validation_uri,:crossvalidation_uri]) unless 
+    Lib::VAL_MERGE_AVG,Lib::VAL_MERGE_SUM,Lib::VAL_MERGE_GENERAL-[:date,:validation_uri,:crossvalidation_uri]) unless 
       Lib::MergeObjects.merge_attributes_registered?(Validation::Validation)
   
-  v = Lib::MergeObjects.merge_array_objects( Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) )
-  v.date = nil
+  #v = Lib::MergeObjects.merge_array_objects( Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) )
+  v = Lib::MergeObjects.merge_array_objects( Validation::Validation.all( :crossvalidation_id => params[:id] ) )
+  v.created_at = nil
   v.id = nil
   
   case request.env['HTTP_ACCEPT'].to_s
@@ -136,12 +148,16 @@ end
 delete '/crossvalidation/:id/?' do
   LOGGER.info "delete crossvalidation with id "+params[:id].to_s
   content_type "text/plain"
-  begin
-    crossvalidation = Validation::Crossvalidation.find(params[:id])
-  rescue ActiveRecord::RecordNotFound => ex
-    halt 404, "Crossvalidation '#{params[:id]}' not found."
-  end
-  Validation::Crossvalidation.delete(params[:id])
+#  begin
+    #crossvalidation = Validation::Crossvalidation.find(params[:id])
+#  rescue ActiveRecord::RecordNotFound => ex
+#    halt 404, "Crossvalidation '#{params[:id]}' not found."
+#  end
+#  Validation::Crossvalidation.delete(params[:id])
+  
+  cv = Validation::Crossvalidation.get(params[:id])
+  halt 404,"Crossvalidation '#{params[:id]}' not found." unless cv
+  cv.delete
 end
 
 #get '/crossvalidation/:id/validations' do
@@ -158,14 +174,16 @@ end
 get '/crossvalidation/:id/predictions' do
   LOGGER.info "get predictions for crossvalidation with id "+params[:id].to_s
   begin
-    crossvalidation = Validation::Crossvalidation.find(params[:id])
+    #crossvalidation = Validation::Crossvalidation.find(params[:id])
+    crossvalidation = Validation::Crossvalidation.get(params[:id])
   rescue ActiveRecord::RecordNotFound => ex
     halt 404, "Crossvalidation '#{params[:id]}' not found."
   end
   halt 400,"Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished
   
   content_type "application/x-yaml"
-  validations = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } )
+  #validations = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } )
+  validations = Validation::Validation.all( :crossvalidation_id => params[:id] )
   p = Lib::OTPredictions.to_array( validations.collect{ |v| v.compute_validation_stats_with_model(nil, true) } ).to_yaml
   
   case request.env['HTTP_ACCEPT'].to_s
@@ -184,8 +202,11 @@ get '/crossvalidation/:id/predictions' do
 end
 
 get '/?' do
+  
   LOGGER.info "list all validations, params: "+params.inspect
-  uri_list = Validation::Validation.find_like(params).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  #uri_list = Validation::Validation.find_like(params).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  uri_list = Validation::Validation.all.collect{ |v| v.validation_uri }.join("\n")+"\n"
+  
   if request.env['HTTP_ACCEPT'] =~ /text\/html/
     related_links = 
       "To perform a validation:\n"+
@@ -216,7 +237,7 @@ post '/test_set_validation' do
   content_type "text/uri-list"
   LOGGER.info "creating test-set-validation "+params.inspect
   if params[:model_uri] and params[:test_dataset_uri] and !params[:training_dataset_uri] and !params[:algorithm_uri]
-    task_uri = OpenTox::Task.as_task( "Perform test-set-validation", url_for("/", :full), params ) do |task|
+    task = OpenTox::Task.create( "Perform test-set-validation", url_for("/", :full) ) do |task| #, params
       v = Validation::Validation.new :validation_type => "test_set_validation", 
                        :model_uri => params[:model_uri], 
                        :test_dataset_uri => params[:test_dataset_uri],
@@ -225,7 +246,7 @@ post '/test_set_validation' do
       v.validate_model( task )
       v.validation_uri
     end
-    halt 202,task_uri+"\n"
+    halt 202,task.uri+"\n"
   else
     halt 400, "illegal parameters, pls specify model_uri and test_dataset_uri\n"+
       "params given: "+params.inspect
@@ -234,7 +255,10 @@ end
 
 get '/test_set_validation' do
   LOGGER.info "list all test-set-validations, params: "+params.inspect
-  uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "test_set_validation" } ).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  
+  #uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "test_set_validation" } ).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  uri_list = Validation::Validation.all( :validation_type => "test_set_validation" ).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  
   if request.env['HTTP_ACCEPT'] =~ /text\/html/
     related_links = 
       "All validations:    "+$sinatra.url_for("/",:full)+"\n"+
@@ -252,10 +276,9 @@ get '/test_set_validation' do
 end
 
 post '/training_test_validation/?' do
-  content_type "text/uri-list"
   LOGGER.info "creating training-test-validation "+params.inspect
   if params[:algorithm_uri] and params[:training_dataset_uri] and params[:test_dataset_uri] and params[:prediction_feature] and !params[:model_uri]
-    task_uri = OpenTox::Task.as_task( "Perform training-test-validation", url_for("/", :full), params ) do |task|
+    task = OpenTox::Task.create( "Perform training-test-validation", url_for("/", :full) ) do |task| #, params
       v = Validation::Validation.new :validation_type => "training_test_validation", 
                         :algorithm_uri => params[:algorithm_uri],
                         :training_dataset_uri => params[:training_dataset_uri], 
@@ -265,7 +288,8 @@ post '/training_test_validation/?' do
       v.validate_algorithm( params[:algorithm_params], task ) 
       v.validation_uri
     end
-    halt 202,task_uri+"\n"
+    content_type "text/uri-list"
+    halt 202,task.uri+"\n"
   else
     halt 400, "illegal parameters, pls specify algorithm_uri, training_dataset_uri, test_dataset_uri, prediction_feature\n"+
         "params given: "+params.inspect
@@ -274,7 +298,8 @@ end
 
 get '/training_test_validation' do
   LOGGER.info "list all training-test-validations, params: "+params.inspect
-  uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "training_test_validation" } ).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  #uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "training_test_validation" } ).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  uri_list = Validation::Validation.all( :validation_type => "training_test_validation" ).collect{ |v| v.validation_uri }.join("\n")+"\n"
   if request.env['HTTP_ACCEPT'] =~ /text\/html/
     related_links = 
       "All validations:    "+$sinatra.url_for("/",:full)+"\n"+
@@ -298,7 +323,7 @@ end
 
 post '/bootstrapping' do
   content_type "text/uri-list"
-  task_uri = OpenTox::Task.as_task( "Perform bootstrapping validation", url_for("/bootstrapping", :full), params ) do |task|
+  task = OpenTox::Task.create( "Perform bootstrapping validation", url_for("/bootstrapping", :full) ) do |task| #, params
     LOGGER.info "performing bootstrapping validation "+params.inspect
     halt 400, "dataset_uri missing" unless params[:dataset_uri]
     halt 400, "algorithm_uri missing" unless params[:algorithm_uri]
@@ -315,12 +340,13 @@ post '/bootstrapping' do
     v.validate_algorithm( params[:algorithm_params], OpenTox::SubTask.create(task,33,100))
     v.validation_uri
   end
-  halt 202,task_uri+"\n"
+  halt 202,task.uri+"\n"
 end
 
 get '/bootstrapping' do
   LOGGER.info "list all bootstrapping-validations, params: "+params.inspect
-  uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "bootstrapping" } ).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  #uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "bootstrapping" } ).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  uri_list = Validation::Validation.all( :validation_type => "bootstrapping" ).collect{ |v| v.validation_uri }.join("\n")+"\n"
   if request.env['HTTP_ACCEPT'] =~ /text\/html/
     related_links = 
       "All validations:    "+$sinatra.url_for("/",:full)+"\n"+
@@ -343,8 +369,7 @@ end
 
 post '/training_test_split' do
   
-  content_type "text/uri-list"
-  task_uri = OpenTox::Task.as_task( "Perform training test split validation", url_for("/training_test_split", :full), params )  do |task|
+  task = OpenTox::Task.create( "Perform training test split validation", url_for("/training_test_split", :full) )  do |task| #, params
     
     LOGGER.info "creating training test split "+params.inspect
     halt 400, "dataset_uri missing" unless params[:dataset_uri]
@@ -362,12 +387,16 @@ post '/training_test_split' do
     v.validate_algorithm( params[:algorithm_params], OpenTox::SubTask.create(task,33,100))
     v.validation_uri
   end
-  halt 202,task_uri+"\n"
+  content_type 'text/uri-list' 
+  halt 503,task.uri+"\n" if task.status == "Cancelled"
+  halt 202,task.uri
+
 end
 
 get '/training_test_split' do
   LOGGER.info "list all training-test-split-validations, params: "+params.inspect
-  uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "training_test_split" } ).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  #uri_list = Validation::Validation.find( :all, :conditions => { :validation_type => "training_test_split" } ).collect{ |v| v.validation_uri }.join("\n")+"\n"
+  uri_list = Validation::Validation.all( :validation_type => "training_test_split" ).collect{ |v| v.validation_uri }.join("\n")+"\n"
   if request.env['HTTP_ACCEPT'] =~ /text\/html/
     related_links = 
       "All validations:    "+$sinatra.url_for("/",:full)+"\n"+
@@ -393,10 +422,12 @@ post '/cleanup/?' do
   LOGGER.info "validation cleanup, starting..."
   content_type "text/uri-list"
   deleted = []
-  Validation::Validation.find( :all, :conditions => { :prediction_dataset_uri => nil } ).each do |val|
-    LOGGER.debug "delete val with id:"+val.id.to_s+" prediction_dataset_uri is nil"
+  #Validation::Validation.find( :all, :conditions => { :prediction_dataset_uri => nil } ).each do |val|
+  Validation::Validation.all( :finished => false ).each do |val|
+    LOGGER.debug "delete val with id:"+val.id.to_s+", finished is false"
     deleted << val.validation_uri
-    Validation::Validation.delete(val.id)
+    #Validation::Validation.delete(val.id)
+    val.delete
   end
   LOGGER.info "validation cleanup, deleted "+deleted.size.to_s+" validations"
   deleted.join("\n")+"\n"
@@ -413,7 +444,7 @@ end
 
 post '/validate_datasets' do
   content_type "text/uri-list"
-  task_uri = OpenTox::Task.as_task( "Perform dataset validation", url_for("/validate_datasets", :full), params ) do |task|
+  task = OpenTox::Task.create( "Perform dataset validation", url_for("/validate_datasets", :full) ) do |task| #, params
     LOGGER.info "validating values "+params.inspect
     halt 400, "test_dataset_uri missing" unless params[:test_dataset_uri]
     halt 400, "prediction_datset_uri missing" unless params[:prediction_dataset_uri]
@@ -429,20 +460,21 @@ post '/validate_datasets' do
             params[:classification] or params[:regression]
       
       predicted_feature = params.delete("predicted_feature")
-      clazz = params.delete("classification")!=nil
-      regr = params.delete("regression")!=nil
+      feature_type = "classification" if params.delete("classification")!=nil
+      feature_type = "regression" if params.delete("regression")!=nil
       v = Validation::Validation.new params            
-      v.compute_validation_stats((clazz and !regr),predicted_feature,nil,false,task)
+      v.compute_validation_stats(feature_type,predicted_feature,nil,nil,false,task)
     end
     v.validation_uri
   end
-  halt 202,task_uri+"\n"
+  halt 202,task.uri+"\n"
 end
 
 get '/:id/predictions' do
   LOGGER.info "get validation predictions "+params.inspect
   begin
-    validation = Validation::Validation.find(params[:id])
+    #validation = Validation::Validation.find(params[:id])
+    validation = Validation::Validation.get(params[:id])
   rescue ActiveRecord::RecordNotFound => ex
     halt 404, "Validation '#{params[:id]}' not found."
   end
@@ -482,7 +514,8 @@ end
 get '/:id' do
   LOGGER.info "get validation with id "+params[:id].to_s+" '"+request.env['HTTP_ACCEPT'].to_s+"'"
   begin
-    validation = Validation::Validation.find(params[:id])
+    #validation = Validation::Validation.find(params[:id])
+    validation = Validation::Validation.get(params[:id])
   rescue ActiveRecord::RecordNotFound => ex
     halt 404, "Validation '#{params[:id]}' not found."
   end
@@ -511,11 +544,13 @@ end
 
 delete '/:id' do
   LOGGER.info "delete validation with id "+params[:id].to_s
-  begin
-    validation = Validation::Validation.find(params[:id])
-  rescue ActiveRecord::RecordNotFound => ex
-    halt 404, "Validation '#{params[:id]}' not found."
-  end
+#  begin
+    #validation = Validation::Validation.find(params[:id])
+#  rescue ActiveRecord::RecordNotFound => ex
+#    halt 404, "Validation '#{params[:id]}' not found."
+#  end
+  validation = Validation::Validation.get(params[:id])
+  halt 404, "Validation '#{params[:id]}' not found." unless validation
   content_type "text/plain"
   validation.delete
 end
\ No newline at end of file
diff --git a/validation/validation_format.rb b/validation/validation_format.rb
index 0db8579..7c6396a 100644..100755
--- a/validation/validation_format.rb
+++ b/validation/validation_format.rb
@@ -24,13 +24,14 @@ module Validation
         h[:crossvalidation_info] = cv
       end
       if classification_statistics 
+        raise "classification_statistics is no has: "+classification_statistics.class.to_s unless classification_statistics.is_a?(Hash)
         clazz = {}
         Lib::VAL_CLASS_PROPS_SINGLE.each{ |p| clazz[p] = classification_statistics[p] }
         
         # transpose results per class
         class_values = {}
         Lib::VAL_CLASS_PROPS_PER_CLASS.each do |p|
-          $sinatra.halt 500, "missing classification statitstics: "+p.to_s+" "+classification_statistics.inspect unless classification_statistics[p]
+          $sinatra.halt 500, "missing classification statitstics: "+p.to_s+" "+classification_statistics.inspect if classification_statistics[p]==nil
           classification_statistics[p].each do |class_value, property_value|
             class_values[class_value] = {:class_value => class_value} unless class_values.has_key?(class_value)
             map = class_values[class_value]
@@ -82,7 +83,8 @@ module Validation
         h[p] = self.send(p)
       end
       v = []
-      Validation.find( :all, :conditions => { :crossvalidation_id => self.id } ).each do |val|
+      #Validation.find( :all, :conditions => { :crossvalidation_id => self.id } ).each do |val|
+      Validation.all( :crossvalidation_id => self.id ).each do |val|
         v.push( val.validation_uri.to_s )
       end
       h[:validation_uris] = v
diff --git a/validation/validation_service.rb b/validation/validation_service.rb
index 7204699..01fba7b 100644..100755
--- a/validation/validation_service.rb
+++ b/validation/validation_service.rb
@@ -1,7 +1,5 @@
 
 
-require "rdf/redland"
-
 require "lib/validation_db.rb"
 require "lib/ot_predictions.rb"
 
@@ -35,13 +33,13 @@ module Validation
   class Validation < Lib::Validation
       
     # constructs a validation object, Rsets id und uri
-    def initialize( params={} )
-      $sinatra.halt 500,"do not set id manually" if params[:id]
-      params[:finished] = false
-      super params
-      self.save!
-      raise "internal error, validation-id not set "+to_yaml if self.id==nil
-    end
+    #def initialize( params={} )
+      #$sinatra.halt 500,"do not set id manually" if params[:id]
+      #params[:finished] = false
+      #super params
+      #self.save!
+      #raise "internal error, validation-id not set "+to_yaml if self.id==nil
+    #end
     
     # deletes a validation
     # PENDING: model and referenced datasets are deleted as well, keep it that way?
@@ -55,7 +53,8 @@ module Validation
         #dataset.delete if dataset
       #end
       
-      Validation::Validation.delete(self.id)
+      #Validation::Validation.delete(self.id)
+      self.destroy
       "Successfully deleted validation "+self.id.to_s+"."
     end
     
@@ -76,14 +75,22 @@ module Validation
       end
       LOGGER.debug "building model '"+algorithm_uri.to_s+"' "+params.inspect
       
-      model = OpenTox::Model::PredictionModel.build(algorithm_uri, params, 
-        OpenTox::SubTask.create(task, 0, 33) )
-      $sinatra.halt 500,"model building failed" unless model
-      self.attributes = { :model_uri => model.uri }
-      self.save!
+      algorithm = OpenTox::Algorithm::Generic.new(algorithm_uri)
+      self.model_uri = algorithm.run(params)
+      task.progress(33)
+      
+      #model = OpenTox::Model::PredictionModel.build(algorithm_uri, params, 
+      #  OpenTox::SubTask.create(task, 0, 33) )
       
-      $sinatra.halt 500,"error after building model: model.dependent_variable != validation.prediciton_feature ("+
-        model.dependentVariables.to_s+" != "+self.prediction_feature+")" if self.prediction_feature!=model.dependentVariables
+      $sinatra.halt 500,"model building failed" unless model_uri
+      #self.attributes = { :model_uri => model_uri }
+      #self.save!
+      
+#      self.save if self.new?
+#      self.update :model_uri => model_uri
+      
+      #$sinatra.halt 500,"error after building model: model.dependent_variable != validation.prediciton_feature ("+
+      #  model.dependentVariables.to_s+" != "+self.prediction_feature+")" if self.prediction_feature!=model.dependentVariables
           
       validate_model OpenTox::SubTask.create(task, 33, 100)
     end
@@ -95,70 +102,103 @@ module Validation
       $sinatra.halt 500,"validation_type missing" unless self.validation_type
       LOGGER.debug "validating model '"+self.model_uri+"'"
       
-      model = OpenTox::Model::PredictionModel.find(self.model_uri)
+      #model = OpenTox::Model::PredictionModel.find(self.model_uri)
+      #$sinatra.halt 400, "model not found: "+self.model_uri.to_s unless model
+      model = OpenTox::Model::Generic.find(self.model_uri)
       $sinatra.halt 400, "model not found: "+self.model_uri.to_s unless model
       
       unless self.algorithm_uri
-        self.attributes = { :algorithm_uri => model.algorithm }
-        self.save!
+#        self.attributes = { :algorithm_uri => model.algorithm }
+#        self.save!
+        #self.update :algorithm_uri => model.algorithm
+        self.algorithm_uri = model.metadata[OT.algorithm]
       end
       
+      dependentVariables = model.metadata[OT.dependentVariables]
       if self.prediction_feature
-        $sinatra.halt 400, "error validating model: model.dependent_variable != validation.prediciton_feature ("+
-          model.dependentVariables+" != "+self.prediction_feature+")" if self.prediction_feature!=model.dependentVariables
+        $sinatra.halt 400, "error validating model: model.dependent_variable != validation.prediction_feature ("+
+          dependentVariables.to_s+" != "+self.prediction_feature+"), model-metadata is "+model.metadata.inspect if self.prediction_feature!=dependentVariables
       else
-        $sinatra.halt 400, "model has no dependentVariables specified, please give prediction feature for model validation" unless model.dependentVariables
-        self.attributes = { :prediction_feature => model.dependentVariables }
-        self.save!
+        $sinatra.halt 400, "model has no dependentVariables specified, please give prediction feature for model validation" unless dependentVariables
+        #self.attributes = { :prediction_feature => model.dependentVariables }
+        #self.save!
+        #self.update :prediction_feature => model.dependentVariables
+        self.prediction_feature = model.metadata[OT.dependentVariables]
       end
       
       prediction_dataset_uri = ""
       benchmark = Benchmark.measure do 
-        prediction_dataset_uri = model.predict_dataset(self.test_dataset_uri, OpenTox::SubTask.create(task, 0, 50))
+        #prediction_dataset_uri = model.predict_dataset(self.test_dataset_uri, OpenTox::SubTask.create(task, 0, 50))
+        prediction_dataset_uri = model.run(:dataset_uri => self.test_dataset_uri)
+        task.progress(50)
       end
-      self.attributes = { :prediction_dataset_uri => prediction_dataset_uri,
-             :real_runtime => benchmark.real }
-      self.save!
-      
+#      self.attributes = { :prediction_dataset_uri => prediction_dataset_uri,
+#             :real_runtime => benchmark.real }
+#      self.save!
+#      self.update :prediction_dataset_uri => prediction_dataset_uri,
+#                  :real_runtime => benchmark.real
+      self.prediction_dataset_uri = prediction_dataset_uri
+      self.real_runtime = benchmark.real
+             
       compute_validation_stats_with_model( model, false, OpenTox::SubTask.create(task, 50, 100) )
     end
       
     def compute_validation_stats_with_model( model=nil, dry_run=false, task=nil )
       
-      model = OpenTox::Model::PredictionModel.find(self.model_uri) if model==nil and self.model_uri
+      #model = OpenTox::Model::PredictionModel.find(self.model_uri) if model==nil and self.model_uri
+      #$sinatra.halt 400, "model not found: "+self.model_uri.to_s unless model
+      model = OpenTox::Model::Generic.find(self.model_uri) if model==nil and self.model_uri
       $sinatra.halt 400, "model not found: "+self.model_uri.to_s unless model
-      prediction_feature = self.prediction_feature ? nil : model.dependentVariables
-      algorithm_uri = self.algorithm_uri ? nil : model.algorithm
-      compute_validation_stats( model.classification?, model.predictedVariables, 
+      
+      dependentVariables = model.metadata[OT.dependentVariables]
+      prediction_feature = self.prediction_feature ? nil : dependentVariables
+      algorithm_uri = self.algorithm_uri ? nil : model.metadata[OT.algorithm]
+      predictedVariables = model.metadata[OT.predictedVariables]
+      compute_validation_stats( model.feature_type, predictedVariables, 
         prediction_feature, algorithm_uri, dry_run, task )
     end
       
-    def compute_validation_stats( classification, predicted_feature, prediction_feature=nil, 
+    def compute_validation_stats( feature_type, predicted_feature, prediction_feature=nil, 
         algorithm_uri=nil, dry_run=false, task=nil )
       
-      self.attributes = { :prediction_feature => prediction_feature } if self.prediction_feature==nil && prediction_feature
-      self.attributes = { :algorithm_uri => algorithm_uri } if self.algorithm_uri==nil && algorithm_uri
-      self.save!
+#      self.attributes = { :prediction_feature => prediction_feature } if self.prediction_feature==nil && prediction_feature
+#      self.attributes = { :algorithm_uri => algorithm_uri } if self.algorithm_uri==nil && algorithm_uri
+#      self.save!
+#      self.update :prediction_feature => prediction_feature if self.prediction_feature==nil && prediction_feature
+#      self.update :algorithm_uri => algorithm_uri if self.algorithm_uri==nil && algorithm_uri
+      self.prediction_feature = prediction_feature if self.prediction_feature==nil && prediction_feature
+      self.algorithm_uri = algorithm_uri if self.algorithm_uri==nil && algorithm_uri
       
       LOGGER.debug "computing prediction stats"
-      prediction = Lib::OTPredictions.new( classification, 
+      prediction = Lib::OTPredictions.new( feature_type, 
         self.test_dataset_uri, self.test_target_dataset_uri, self.prediction_feature, 
         self.prediction_dataset_uri, predicted_feature, OpenTox::SubTask.create(task, 0, 80) )
       #reading datasets and computing the main stats is 80% the work 
-        
+      
       unless dry_run
-        if prediction.classification?
-          self.attributes = { :classification_statistics => prediction.compute_stats }
-        else
-          self.attributes = { :regression_statistics => prediction.compute_stats }
+        case feature_type
+        when "classification"
+          #self.attributes = { :classification_statistics => prediction.compute_stats }
+          #self.update :classification_statistics => prediction.compute_stats 
+          self.classification_statistics = prediction.compute_stats
+        when "regression"
+          #self.attributes = { :regression_statistics => prediction.compute_stats }
+          self.regression_statistics = prediction.compute_stats
         end
-        self.attributes = { :num_instances => prediction.num_instances,
+#        self.attributes = { :num_instances => prediction.num_instances,
+#               :num_without_class => prediction.num_without_class,
+#               :percent_without_class => prediction.percent_without_class,
+#               :num_unpredicted => prediction.num_unpredicted,
+#               :percent_unpredicted => prediction.percent_unpredicted,
+#               :finished => true}
+#        self.save!
+        self.attributes= {:num_instances => prediction.num_instances,
                :num_without_class => prediction.num_without_class,
                :percent_without_class => prediction.percent_without_class,
                :num_unpredicted => prediction.num_unpredicted,
                :percent_unpredicted => prediction.percent_unpredicted,
                :finished => true}
-        self.save!
+         self.save
       end
       
       task.progress(100) if task
@@ -169,17 +209,17 @@ module Validation
   class Crossvalidation < Lib::Crossvalidation
     
     # constructs a crossvalidation, id and uri are set
-    def initialize( params={} )
-      
-      $sinatra.halt 500,"do not set id manually" if params[:id]
-      params[:num_folds] = 10 if params[:num_folds]==nil
-      params[:random_seed] = 1 if params[:random_seed]==nil
-      params[:stratified] = false if params[:stratified]==nil
-      params[:finished] = false
-      super params
-      self.save!
-      raise "internal error, crossvalidation-id not set" if self.id==nil
-    end
+    #def initialize( params={} )
+    #  
+    #  $sinatra.halt 500,"do not set id manually" if params[:id]
+    #  params[:num_folds] = 10 if params[:num_folds]==nil
+    #  params[:random_seed] = 1 if params[:random_seed]==nil
+    #  params[:stratified] = false if params[:stratified]==nil
+    #  params[:finished] = false
+    #  super params
+    #  self.save!
+    #  raise "internal error, crossvalidation-id not set" if self.id==nil
+    #end
     
     def perform_cv ( prediction_feature, algorithm_params=nil, task=nil )
       
@@ -189,7 +229,7 @@ module Validation
     
     # deletes a crossvalidation, all validations are deleted as well
     def delete
-        Validation.all(:crossvalidation_id => self.id).each{ |v| v.delete }
+        Validation.all(:crossvalidation_id => self.id).each{ |v| v.destroy }
         destroy
         "Successfully deleted crossvalidation "+self.id.to_s+"."
     end
@@ -216,8 +256,11 @@ module Validation
         i += 1
       end
       
-      self.attributes = { :finished => true }
-      self.save!      
+#      self.attributes = { :finished => true }
+#      self.save!
+      #self.save if self.new?
+      self.finished = true
+      self.save
     end
     
     private
@@ -255,7 +298,7 @@ module Validation
     # stores uris in validation objects 
     def create_new_cv_datasets( prediction_feature, task = nil )
       
-      $sinatra.halt(500,"random seed not set") unless self.random_seed
+      $sinatra.halt(500,"random seed not set "+self.inspect) unless self.random_seed
       LOGGER.debug "creating datasets for crossvalidation"
       orig_dataset = OpenTox::Dataset.find(self.dataset_uri)
       $sinatra.halt 400, "Dataset not found: "+self.dataset_uri.to_s unless orig_dataset
@@ -302,7 +345,7 @@ module Validation
       end
       LOGGER.debug "cv: num instances for each fold: "+split_compounds.collect{|c| c.size}.join(", ")
       
-      test_features = orig_dataset.features.dclone - [prediction_feature]
+      test_features = orig_dataset.features.keys.dclone - [prediction_feature]
       
       @tmp_validations = []
       
@@ -332,11 +375,17 @@ module Validation
         $sinatra.halt 500,"internal error, num train compounds not correct" unless shuffled_compounds.size - test_compounds.size == train_compounds.size
         
         LOGGER.debug "training set: "+datasetname+"_train, compounds: "+train_compounds.size.to_s
-        train_dataset_uri = orig_dataset.create_new_dataset( train_compounds, orig_dataset.features, datasetname + '_train', source ) 
+        #train_dataset_uri = orig_dataset.create_new_dataset( train_compounds, orig_dataset.features, datasetname + '_train', source ) 
+        train_dataset_uri = orig_dataset.split( train_compounds, orig_dataset.features.keys, 
+          { DC.title => datasetname + '_train', DC.creator => source } ).uri
         
         LOGGER.debug "test set:     "+datasetname+"_test, compounds: "+test_compounds.size.to_s
-        test_dataset_uri = orig_dataset.create_new_dataset( test_compounds, test_features, datasetname + '_test', source )
-      
+        #test_dataset_uri = orig_dataset.create_new_dataset( test_compounds, test_features, datasetname + '_test', source )
+        test_dataset_uri = orig_dataset.split( test_compounds, test_features, 
+          { DC.title => datasetname + '_test', DC.creator => source } ).uri
+        
+        #make sure self.id is set
+        self.save if self.new?
         tmp_validation = { :validation_type => "crossvalidation",
                            :training_dataset_uri => train_dataset_uri, 
                            :test_dataset_uri => test_dataset_uri,
@@ -362,6 +411,7 @@ module Validation
       random_seed=1 unless random_seed
       
       orig_dataset = OpenTox::Dataset.find orig_dataset_uri
+      orig_dataset.load_all
       $sinatra.halt 400, "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset
       if prediction_feature
         $sinatra.halt 400, "Prediction feature '"+prediction_feature.to_s+
@@ -374,6 +424,11 @@ module Validation
       compounds = orig_dataset.compounds
       $sinatra.halt 400, "Cannot split datset, num compounds in dataset < 2 ("+compounds.size.to_s+")" if compounds.size<2
       
+      compounds.each do |c|
+        $sinatra.halt 400, "Bootstrapping not yet implemented for duplicate compounds" if
+          orig_dataset.data_entries[c][prediction_feature].size > 1
+      end
+      
       srand random_seed.to_i
       while true
         training_compounds = []
@@ -397,23 +452,35 @@ module Validation
       task.progress(33) if task
       
       result = {}
-      result[:training_dataset_uri] = orig_dataset.create_new_dataset( training_compounds,
-        orig_dataset.features, 
-        "Bootstrapping training dataset of "+orig_dataset.title.to_s, 
-        $sinatra.url_for('/bootstrapping',:full) )
+#      result[:training_dataset_uri] = orig_dataset.create_new_dataset( training_compounds,
+#        orig_dataset.features, 
+#        "Bootstrapping training dataset of "+orig_dataset.title.to_s, 
+#        $sinatra.url_for('/bootstrapping',:full) )
+      result[:training_dataset_uri] = orig_dataset.split( training_compounds,
+        orig_dataset.features.keys, 
+        { DC.title => "Bootstrapping training dataset of "+orig_dataset.title.to_s,
+          DC.creator => $sinatra.url_for('/bootstrapping',:full) }).uri
       task.progress(66) if task
 
-      result[:test_dataset_uri] = orig_dataset.create_new_dataset( test_compounds,
-        orig_dataset.features.dclone - [prediction_feature], 
-        "Bootstrapping test dataset of "+orig_dataset.title.to_s, 
-        $sinatra.url_for('/bootstrapping',:full) )
+#      result[:test_dataset_uri] = orig_dataset.create_new_dataset( test_compounds,
+#        orig_dataset.features.dclone - [prediction_feature], 
+#        "Bootstrapping test dataset of "+orig_dataset.title.to_s, 
+#        $sinatra.url_for('/bootstrapping',:full) )
+      result[:test_dataset_uri] = orig_dataset.split( test_compounds,
+        orig_dataset.features.keys.dclone - [prediction_feature],
+        { DC.title => "Bootstrapping test dataset of "+orig_dataset.title.to_s,
+          DC.creator => $sinatra.url_for('/bootstrapping',:full) }).uri
       task.progress(100) if task
       
       if ENV['RACK_ENV'] =~ /test|debug/
         training_dataset = OpenTox::Dataset.find result[:training_dataset_uri]
         $sinatra.halt 400, "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless training_dataset
-        training_compounds_verify = training_dataset.compounds
-        $sinatra.halt 500, "training compounds error" unless training_compounds_verify==training_compounds
+        training_dataset.load_all
+        value_count = 0
+        training_dataset.compounds.each do |c|
+          value_count += training_dataset.data_entries[c][prediction_feature].size
+        end
+        $sinatra.halt 500, "training compounds error" unless value_count==training_compounds.size
         $sinatra.halt 400, "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless OpenTox::Dataset.find result[:test_dataset_uri]
       end
       LOGGER.debug "bootstrapping done, training dataset: '"+result[:training_dataset_uri].to_s+"', test dataset: '"+result[:test_dataset_uri].to_s+"'"
@@ -429,13 +496,14 @@ module Validation
       random_seed=1 unless random_seed
       
       orig_dataset = OpenTox::Dataset.find orig_dataset_uri
+      orig_dataset.load_all
       $sinatra.halt 400, "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset
       $sinatra.halt 400, "Split ratio invalid: "+split_ratio.to_s unless split_ratio and split_ratio=split_ratio.to_f
       $sinatra.halt 400, "Split ratio not >0 and <1 :"+split_ratio.to_s unless split_ratio>0 && split_ratio<1
       if prediction_feature
         $sinatra.halt 400, "Prediction feature '"+prediction_feature.to_s+
           "' not found in dataset, features are: \n"+
-          orig_dataset.features.inspect unless orig_dataset.features.include?(prediction_feature)
+          orig_dataset.features.keys.inspect unless orig_dataset.features.include?(prediction_feature)
       else
         LOGGER.warn "no prediciton feature given, all features included in test dataset"
       end
@@ -453,21 +521,46 @@ module Validation
       task.progress(33) if task
       
       result = {}
-      result[:training_dataset_uri] = orig_dataset.create_new_dataset( compounds[0..split],
-        orig_dataset.features, 
-        "Training dataset split of "+orig_dataset.title.to_s, 
-        $sinatra.url_for('/training_test_split',:full) )
+#      result[:training_dataset_uri] = orig_dataset.create_new_dataset( compounds[0..split],
+#        orig_dataset.features, 
+#        "Training dataset split of "+orig_dataset.title.to_s, 
+#        $sinatra.url_for('/training_test_split',:full) )
+
+#      orig_dataset.data_entries.each do |k,v|
+#        puts k.inspect+" =>"+v.inspect
+#        puts v.values[0].to_s+" "+v.values[0].class.to_s
+#      end
+
+      result[:training_dataset_uri] = orig_dataset.split( compounds[0..split],
+        orig_dataset.features.keys, 
+        { DC.title => "Training dataset split of "+orig_dataset.title.to_s, 
+          DC.creator => $sinatra.url_for('/training_test_split',:full) } ).uri
       task.progress(66) if task
-      
-      result[:test_dataset_uri] = orig_dataset.create_new_dataset( compounds[(split+1)..-1],
-        orig_dataset.features.dclone - [prediction_feature], 
-        "Test dataset split of "+orig_dataset.title.to_s, 
-        $sinatra.url_for('/training_test_split',:full) )
+
+#      d = OpenTox::Dataset.find(result[:training_dataset_uri])
+#      d.data_entries.values.each do |v|
+#        puts v.inspect
+#        puts v.values[0].to_s+" "+v.values[0].class.to_s
+#      end
+#      raise "stop here"
+      
+#      result[:test_dataset_uri] = orig_dataset.create_new_dataset( compounds[(split+1)..-1],
+#        orig_dataset.features.dclone - [prediction_feature], 
+#        "Test dataset split of "+orig_dataset.title.to_s, 
+#        $sinatra.url_for('/training_test_split',:full) )
+      result[:test_dataset_uri] = orig_dataset.split( compounds[(split+1)..-1],
+        orig_dataset.features.keys.dclone - [prediction_feature], 
+        { DC.title => "Test dataset split of "+orig_dataset.title.to_s, 
+          DC.creator => $sinatra.url_for('/training_test_split',:full) } ).uri
       task.progress(100) if task  
       
       if ENV['RACK_ENV'] =~ /test|debug/
         $sinatra.halt 400, "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless OpenTox::Dataset.find result[:training_dataset_uri]
-        $sinatra.halt 400, "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless OpenTox::Dataset.find result[:test_dataset_uri]
+        test_data = OpenTox::Dataset.find result[:test_dataset_uri]
+        $sinatra.halt 400, "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless test_data 
+        test_data.load_compounds
+        $sinatra.halt 400, "Test dataset num coumpounds != "+(compounds.size-split-1).to_s+", instead: "+
+          test_data.compounds.size.to_s+"\n"+test_data.to_yaml unless test_data.compounds.size==(compounds.size-1-split)
       end
       
       LOGGER.debug "split done, training dataset: '"+result[:training_dataset_uri].to_s+"', test dataset: '"+result[:test_dataset_uri].to_s+"'"
diff --git a/validation/validation_test.rb b/validation/validation_test.rb
index b1fbf25..d9c2fbb 100644..100755
--- a/validation/validation_test.rb
+++ b/validation/validation_test.rb
@@ -7,7 +7,7 @@ require 'rack/test'
 require 'lib/test_util.rb'
 require 'test/test_examples.rb'
 
-LOGGER = MyLogger.new(STDOUT)
+LOGGER = OTLogger.new(STDOUT)
 LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S "
 LOGGER.formatter = Logger::Formatter.new
 
@@ -15,7 +15,7 @@ LOGGER.formatter = Logger::Formatter.new
 module Sinatra
   module UrlForHelper
     BASE = "http://localhost/validation"
-   def url_for url_fragment, mode=:path_only
+    def url_for url_fragment, mode=:path_only
       case mode
       when :path_only
         raise "not impl"
@@ -35,8 +35,9 @@ class ValidationTest < Test::Unit::TestCase
     $test_case = self
     
     #get "/1",nil,'HTTP_ACCEPT' => "text/html" 
-#    get "/",nil,'HTTP_ACCEPT' => "text/html" 
+    #get "/2",nil,'HTTP_ACCEPT' => "application/rdf+xml" 
     #puts last_response.body
+    #exit
     
 #    d = OpenTox::Dataset.find("http://ot-dev.in-silico.ch/dataset/307")
 #    puts d.compounds.inspect
@@ -59,12 +60,13 @@ class ValidationTest < Test::Unit::TestCase
     #post "/crossvalidation/cleanup"
     #puts last_response.body
 
-    #get "/crossvalidation/4/statistics"
+    #get "/crossvalidation/19/predictions",nil,'HTTP_ACCEPT' => "application/x-yaml" #/statistics"
 #    post "",:model_uri=>"http://localhost/model/1",:test_dataset_uri=>"http://localhost/dataset/3",
 #      :test_target_dataset_uri=>"http://localhost/dataset/1"
 
 #    get "/crossvalidation/2",nil,'HTTP_ACCEPT' => "application/rdf+xml" 
-    #puts last_response.body
+   #puts last_response.body
+   #exit
     
     #get "/crossvalidation?model_uri=lazar"
 #    post "/test_validation",:select=>"6d" #,:report=>"yes,please"
@@ -79,8 +81,32 @@ class ValidationTest < Test::Unit::TestCase
 #      :regression=>"true"}
 #      #:classification=>"true"}
 #    puts last_response.body
-    
-    #run_test("1b","http://localhost/validation/28")#,"http://localhost/validation/394");
+
+#     post "/validate_datasets",{
+#      :test_dataset_uri=>"http://localhost/dataset/89",
+#       :prediction_dataset_uri=>"http://localhost/dataset/91",
+#       :test_target_dataset_uri=>"http://localhost/dataset/87",
+#       :prediction_feature=>"http://localhost/dataset/1/feature/hamster_carcinogenicity",
+#       :predicted_feature=>"",
+##      :regression=>"true"}
+#       :classification=>"true"}
+#    puts last_response.body
+
+    # m = OpenTox::Model::Generic.find("http://localhost/model/1323333")
+    # puts m.to_yaml
+
+#     post "/validate_datasets",{
+#      :test_dataset_uri=>"http://localhost/dataset/150",
+#       :prediction_dataset_uri=>"http://localhost/dataset/152",
+#       :test_target_dataset_uri=>"http://localhost/dataset/148",
+#       :prediction_feature=>"http://localhost/dataset/148/feature/LC50_mmol",
+#       :model_uri=>"http://localhost/model/13"}
+#      #:regression=>"true"}
+##       :classification=>"true"}
+#    puts last_response.body
+
+    #run_test("13a","http://localhost/validation/39",nil,false) #,"http://localhost/validation/28")#,"http://localhost/validation/394");
+    run_test("1b",nil,nil,false)
     
     #run_test("7b","http://localhost/validation/21")
     
@@ -96,7 +122,7 @@ class ValidationTest < Test::Unit::TestCase
  
     #puts Nightly.build_nightly("1")
     
-   prepare_examples
+    #prepare_examples
     #do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE
     #do_test_examples_ortona
   end
@@ -105,7 +131,7 @@ class ValidationTest < Test::Unit::TestCase
     Sinatra::Application
   end
   
-  def run_test(select=nil, validation_uri=nil, report_uri=nil)
+  def run_test(select=nil, validation_uri=nil, report_uri=nil, delete=false)
     puts ValidationExamples.list unless select
     validationExamples = ValidationExamples.select(select)
     validationExamples.each do |vv|
@@ -122,8 +148,9 @@ class ValidationTest < Test::Unit::TestCase
         unless ex.report_uri
           ex.report
         end
-        ex.verify_yaml
-        ex.compare_yaml_vs_rdf
+        #ex.verify_yaml
+        #ex.compare_yaml_vs_rdf
+        ex.delete if delete
       end
     end
   end