new feature: algorithm comparison report

15 files changed, 599 insertions, 285 deletions

diff --git a/lib/merge.rb b/lib/merge.rb
index ecbe133..f30a3c1 100644
--- a/lib/merge.rb
+++ b/lib/merge.rb
@@ -126,7 +126,7 @@ module Lib
           if value1==nil && value2==nil
             value = nil
           elsif value1.to_s != value2.to_s
-            value = value1.to_s + "/" + value2.to_s
+            value = value1.to_s + ";" + value2.to_s
           else
             value = value2.to_s
           end
diff --git a/lib/predictions.rb b/lib/predictions.rb
index 420790e..f97b764 100755
--- a/lib/predictions.rb
+++ b/lib/predictions.rb
@@ -170,8 +170,8 @@ module Lib
     
     def percent_correct
       raise "no classification" unless @feature_type=="classification"
-      return 0 if @num_with_actual_value==0
-      return 100 * @num_correct / (@num_with_actual_value - @num_unpredicted).to_f
+      pct = 100 * @num_correct / (@num_with_actual_value - @num_unpredicted).to_f
+      pct.nan? ? 0 : pct 
     end
     
     def percent_incorrect
@@ -181,7 +181,8 @@ module Lib
     end
     
     def accuracy
-      return percent_correct / 100.0
+      acc = percent_correct / 100.0
+      acc.nan? ? 0 : acc
     end
     
     def weighted_accuracy
@@ -250,6 +251,7 @@ module Lib
       return res
     end
     
+    # does only take the instances that are classified as <class-index> into account
     def area_under_roc(class_index=nil)
       return prediction_feature_value_map( lambda{ |i| area_under_roc(i) } ) if 
         class_index==nil
@@ -427,8 +429,13 @@ module Lib
       return incorrect
     end
     
+    # Note:
+    # * (un-weighted) area under roc is computed with all __predicted__ isntances for a certain class
+    # * weighted weights each auc with the number of __acutal__ instances
+    # its like that, because its like that in weka   
     def weighted_area_under_roc
-      return weighted_measure( :area_under_roc )
+      w_auc = weighted_measure( :area_under_roc )
+      w_auc.nan? ? 0 : w_auc
     end
     
     def weighted_f_measure
@@ -436,6 +443,7 @@ module Lib
     end
     
     private
+    # the <measure> is weighted with the number of instances for each actual class value 
     def weighted_measure( measure )
       
       sum_instances = 0
@@ -478,9 +486,10 @@ module Lib
     
     def sample_correlation_coefficient
       # formula see http://en.wikipedia.org/wiki/Correlation_and_dependence#Pearson.27s_product-moment_coefficient
-      return ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) /
+      scc = ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) /
              ( Math.sqrt( [0, @num_predicted * @sum_squares_actual - @sum_actual**2].max ) *
                Math.sqrt( [0, @num_predicted * @sum_squares_predicted - @sum_predicted**2].max ) )
+      ( scc.infinite? || scc.nan? ) ? 0 : scc
     end
     
     def total_sum_of_squares
diff --git a/report/plot_factory.rb b/report/plot_factory.rb
index b7c920a..74c89f5 100644
--- a/report/plot_factory.rb
+++ b/report/plot_factory.rb
@@ -167,25 +167,35 @@ module Reports
       validation_set.validations.each do |v|
         values = []
         value_attributes.each do |a|
-          validation_set.get_accept_values_for_attr(a).each do |class_value|
-            value = v.send(a)
-            if value.is_a?(Hash)
-              if class_value==nil
-                avg_value = 0
-                value.values.each{ |val| avg_value+=val }
-                value = avg_value/value.values.size.to_f
-              else
-                raise "bar plot value is hash, but no entry for class-value ("+class_value.to_s+"); value for "+a.to_s+" -> "+value.inspect unless value.key?(class_value)
-                value = value[class_value]
+          
+          accept = validation_set.get_accept_values_for_attr(a)
+          if accept and accept.size>0
+            accept.each do |class_value|
+              value = v.send(a)
+              if value.is_a?(Hash)
+                if class_value==nil
+                  avg_value = 0
+                  value.values.each{ |val| avg_value+=val }
+                  value = avg_value/value.values.size.to_f
+                else
+                  raise "bar plot value is hash, but no entry for class-value ("+class_value.to_s+"); value for "+a.to_s+" -> "+value.inspect unless value.key?(class_value)
+                  value = value[class_value]
+                end
               end
+              raise "value is nil\nattribute: "+a.to_s+"\nvalidation: "+v.inspect if value==nil
+              values.push(value)
+              labels.push(a.to_s.gsub("_","-") + ( class_value==nil ? "" : "("+class_value.to_s+")" ))
             end
-            raise "value is nil\nattribute: "+a.to_s+"\nvalidation: "+v.inspect if value==nil
+          else
+            value = v.send(a)
             values.push(value)
-            labels.push(a.to_s.gsub("_","-") + ( class_value==nil ? "" : "("+class_value.to_s+")" ))
+            labels.push(a.to_s.gsub("_","-"))
           end
+          
         end
         
         titles << v.send(title_attribute).to_s
+        raise "no title for '"+title_attribute.to_s+"' in validation: "+v.to_yaml if titles[-1].to_s.size==0
         data << values
       end
       
diff --git a/report/report_application.rb b/report/report_application.rb
index 258daa7..1637ead 100755
--- a/report/report_application.rb
+++ b/report/report_application.rb
@@ -114,7 +114,8 @@ end
 post '/report/:type' do
   task = OpenTox::Task.create("Create report",url_for("/report/"+params[:type], :full)) do |task| #,params
     perform do |rs|
-      rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil,@subjectid,task)
+      rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil,
+        params[:identifier]?params[:identifier].split(/\n|,/):nil,@subjectid,task)
     end
   end
   return_task(task)
diff --git a/report/report_content.rb b/report/report_content.rb
index cc4c13c..ac64bab 100755
--- a/report/report_content.rb
+++ b/report/report_content.rb
@@ -22,28 +22,32 @@ class Reports::ReportContent
     @current_section = @xml_report.get_root_element
   end
   
-  def add_paired_ttest_table( validation_set,
+  def add_paired_ttest_tables( validation_set,
                        group_attribute, 
-                       test_attribute,
+                       test_attributes,
                        section_title = "Paired t-test",
                        section_text = nil)
-                       
-    level = 0.90                       
-    test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, 
-      group_attribute, test_attribute, "paired_ttest", level )
-    #puts test_matrix.inspect
-    titles = test_matrix[:titles]
-    matrix = test_matrix[:matrix]
-    table = []
-    #puts titles.inspect
-    table << [""] + titles
-    titles.size.times do |i|
-      table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") }
-    end
-    
+
     section_test = @xml_report.add_section(@current_section, section_title)
     @xml_report.add_paragraph(section_test, section_text) if section_text
-    @xml_report.add_table(section_test, test_attribute.to_s+", significance-level: "+level.to_s, table, true, true)  
+      
+    test_attributes.each do |test_attribute|                    
+      level = 0.90                       
+      test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, 
+        group_attribute, test_attribute, "paired_ttest", level )
+      #puts test_matrix.inspect
+      titles = test_matrix[:titles]
+      matrix = test_matrix[:matrix]
+      table = []
+      #puts titles.inspect
+      table << [""] + titles
+      titles.size.times do |i|
+        table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") }
+      end
+      
+      @xml_report.add_table(section_test, test_attribute.to_s+", significance-level: "+level.to_s+", num results: "+
+        test_matrix[:num_results].to_s, table, true, true)
+    end
   end
   
   def add_predictions( validation_set, 
@@ -98,32 +102,13 @@ class Reports::ReportContent
                         validation_attributes,
                         table_title,
                         section_title="Results",
-                        section_text=nil,
-                        #rem_equal_vals_attr=[],
-                        search_for_existing_report_type=nil)
+                        section_text=nil)
+                        #rem_equal_vals_attr=[])
 
     section_table = @xml_report.add_section(@current_section, section_title)
     @xml_report.add_paragraph(section_table, section_text) if section_text
     vals = validation_set.to_array(validation_attributes, true)
     vals = vals.collect{|a| a.collect{|v| v.to_s }}
-    
-    if (search_for_existing_report_type)
-      vals.size.times do |i|
-        #puts i
-        if (i==0)
-          vals[i] = [ "Reports" ] + vals[i]
-          puts vals[i].inspect
-        else
-          if search_for_existing_report_type=="validation"
-            vals[i] = [ validation_set.validations[i-1].validation_report_uri() ] + vals[i]
-          elsif search_for_existing_report_type=="crossvalidation"
-            vals[i] = [ validation_set.validations[i-1].cv_report_uri() ] + vals[i]
-          else
-            raise "illegal report type: "+search_for_existing_report_type.to_s
-          end
-        end
-      end
-    end
     #PENDING transpose values if there more than 4 columns, and there are more than columns than rows
     transpose = vals[0].size>4 && vals[0].size>vals.size
     @xml_report.add_table(section_table, table_title, vals, !transpose, transpose, transpose)
diff --git a/report/report_factory.rb b/report/report_factory.rb
index 08d9418..7e74cb4 100755
--- a/report/report_factory.rb
+++ b/report/report_factory.rb
@@ -100,33 +100,37 @@ module Reports::ReportFactory
     raise OpenTox::BadRequestError.new("validations must have unique feature type, i.e. must be either all regression, "+
       +"or all classification validations") unless validation_set.unique_feature_type  
     pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) )
+    validation_set.validations.sort! do |x,y|
+      x.crossvalidation_fold.to_f <=> y.crossvalidation_fold.to_f
+    end
+    cv_set = validation_set.replace_with_cv_stats
+    raise unless cv_set.size==1
     
-    merged = validation_set.merge([:crossvalidation_id])
-    raise unless merged.size==1
-    
-    #puts merged.get_values(:percent_correct_variance, false).inspect
+    #puts cv_set.get_values(:percent_correct_variance, false).inspect
     report = Reports::ReportContent.new("Crossvalidation report")
+    res_titel = "Crossvalidation Results"
+    res_text = "These performance statistics have been derieved by accumulating all predictions on the various fold (i.e. these numbers are NOT averaged results over all crossvalidation folds)."
     
     case validation_set.unique_feature_type
     when "classification"
-      report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold],"Mean Results","Mean Results")
-      report.add_confusion_matrix(merged.validations[0])
+      report.add_result(cv_set, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold], res_titel, res_titel, res_text)
+      report.add_confusion_matrix(cv_set.validations[0])
       report.add_section("Plots")
       report.add_roc_plot(validation_set)
       report.add_roc_plot(validation_set, :crossvalidation_fold)
       report.add_confidence_plot(validation_set)
       report.add_confidence_plot(validation_set, :crossvalidation_fold)
       report.end_section
-      report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_CLASS-[:num_folds],
-        "Results","Results",nil,"validation")
+      report.add_result(validation_set, [:validation_uri, :validation_report_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:num_folds, :dataset_uri, :algorithm_uri],
+        "Results","Results")
     when "regression"
-      report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold],"Mean Results","Mean Results")
+      report.add_result(cv_set, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold],res_titel, res_titel, res_text)
       report.add_section("Plots")
       report.add_regression_plot(validation_set, :crossvalidation_fold)
       report.add_confidence_plot(validation_set)
       report.add_confidence_plot(validation_set, :crossvalidation_fold)
       report.end_section
-      report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_REGR-[:num_folds], "Results","Results")
+      report.add_result(validation_set, [:validation_uri, :validation_report_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:num_folds, :dataset_uri, :algorithm_uri], "Results","Results")
     end
     task.progress(90) if task
       
@@ -142,8 +146,8 @@ module Reports::ReportFactory
     raise OpenTox::BadRequestError.new("num validations is not >1") unless validation_set.size>1
     raise OpenTox::BadRequestError.new("validations must have unique feature type, i.e. must be either all regression, "+
       +"or all classification validations") unless validation_set.unique_feature_type
-    raise OpenTox::BadRequestError.new("number of different algorithms <2: "+
-      validation_set.get_values(:algorithm_uri).inspect) if validation_set.num_different_values(:algorithm_uri)<2
+    raise OpenTox::BadRequestError.new("number of different identifiers <2: "+
+      validation_set.get_values(:identifier).inspect) if validation_set.num_different_values(:identifier)<2
       
     if validation_set.has_nil_values?(:crossvalidation_id)
       raise OpenTox::BadRequestError.new("algorithm comparison for non crossvalidation not yet implemented")
@@ -160,73 +164,63 @@ module Reports::ReportFactory
     
     # groups results into sets with equal dataset 
     if (validation_set.num_different_values(:dataset_uri)>1)
+      LOGGER.debug "compare report -- num different datasets: "+validation_set.num_different_values(:dataset_uri).to_s
       dataset_grouping = Reports::Util.group(validation_set.validations, [:dataset_uri])
       # check if equal values in each group exist
-      Reports::Util.check_group_matching(dataset_grouping, [:algorithm_uri, :crossvalidation_fold, :num_folds, :stratified, :random_seed])
+      Reports::Util.check_group_matching(dataset_grouping, [:crossvalidation_fold, :num_folds, :stratified, :random_seed])
     else
       dataset_grouping = [ validation_set.validations ]
     end
     
-    # we only checked that equal validations exist in each dataset group, now check for each algorithm
+    # we only checked that equal validations exist in each dataset group, now check for each identifier
     dataset_grouping.each do |validations|
-      algorithm_grouping = Reports::Util.group(validations, [:algorithm_uri])
+      algorithm_grouping = Reports::Util.group(validations, [:identifier])
       Reports::Util.check_group_matching(algorithm_grouping, [:crossvalidation_fold, :num_folds, :stratified, :random_seed])
     end
     
     pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) )
-    report = Reports::ReportContent.new("Algorithm comparison report - Many datasets")
+    report = Reports::ReportContent.new("Algorithm comparison report")
     
     if (validation_set.num_different_values(:dataset_uri)>1)
       all_merged = validation_set.merge([:algorithm_uri, :dataset_uri, :crossvalidation_id, :crossvalidation_uri])
       report.add_ranking_plots(all_merged, :algorithm_uri, :dataset_uri,
         [:percent_correct, :weighted_area_under_roc, :true_positive_rate, :true_negative_rate] )
       report.add_result_overview(all_merged, :algorithm_uri, :dataset_uri, [:percent_correct, :weighted_area_under_roc, :true_positive_rate, :true_negative_rate])
-      
     end
-
+      
+    result_attributes = [:identifier,:crossvalidation_uri,:crossvalidation_report_uri]+VAL_ATTR_CV-[:crossvalidation_fold,:num_folds,:dataset_uri]
     case validation_set.unique_feature_type
     when "classification"
-      attributes = VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold]
-      attributes = ([ :dataset_uri ] + attributes).uniq
-      
-      dataset_grouping.each do |validations|
-      
-        set = Reports::ValidationSet.create(validations)
-        
-        dataset = validations[0].dataset_uri
-        merged = set.merge([:algorithm_uri, :dataset_uri, :crossvalidation_id, :crossvalidation_uri])
-        merged.sort(:dataset_uri)
-        
-        report.add_section("Dataset: "+dataset)
-        report.add_result(merged,attributes,
-          "Mean Results","Mean Results",nil,"crossvalidation")
-        report.add_paired_ttest_table(set, :algorithm_uri, :percent_correct)
-        
-        report.add_bar_plot(merged, :algorithm_uri, VAL_ATTR_BAR_PLOT_CLASS)
-        report.add_roc_plot(set, :algorithm_uri)
-        report.end_section
-      end
-      
-    when "regression"
+      result_attributes += VAL_ATTR_CLASS
+      ttest_attributes = [:percent_correct, :weighted_area_under_roc]
+      bar_plot_attributes = VAL_ATTR_BAR_PLOT_CLASS
+    else 
+      result_attributes += VAL_ATTR_REGR
+      ttest_attributes = [:r_square, :root_mean_squared_error]
+      bar_plot_attributes = VAL_ATTR_BAR_PLOT_REGR
+    end
       
-      attributes = VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold]
-      attributes = ([ :dataset_uri ] + attributes).uniq
+    dataset_grouping.each do |validations|
+    
+      set = Reports::ValidationSet.create(validations)
       
-      dataset_grouping.each do |validations|
+      dataset = validations[0].dataset_uri
+      merged = set.merge([:identifier, :dataset_uri]) #, :crossvalidation_id, :crossvalidation_uri])
+      merged.sort(:identifier)
       
-        set = Reports::ValidationSet.create(validations)
-        
-        dataset = validations[0].dataset_uri
-        merged = set.merge([:algorithm_uri, :dataset_uri, :crossvalidation_id, :crossvalidation_uri])
-        merged.sort(:dataset_uri)
-        
-        report.add_section("Dataset: "+dataset)
-        report.add_result(merged,attributes,
-          "Mean Results","Mean Results",nil,"crossvalidation")
-        report.add_paired_ttest_table(set, :algorithm_uri, :r_square)
-        report.end_section
+      merged.validations.each do |v|
+        v.crossvalidation_uri = v.crossvalidation_uri.split(";").uniq.join(" ")
+        v.crossvalidation_report_uri = v.crossvalidation_report_uri.split(";").uniq.join(" ") if  v.crossvalidation_report_uri
       end
       
+      report.add_section("Dataset: "+dataset)
+      res_titel = "Average Results on Folds"
+      res_text = "These performance statistics have been derieved by computing the mean of the statistics on each crossvalidation fold."
+      report.add_result(merged,result_attributes,res_titel,res_titel,res_text)
+      # pending: regression stats have different scales!!!
+      report.add_bar_plot(merged, :identifier, bar_plot_attributes) if validation_set.unique_feature_type=="classification"
+      report.add_paired_ttest_tables(set, :identifier, ttest_attributes)
+      report.end_section
     end
     task.progress(100) if task
     report
diff --git a/report/report_persistance.rb b/report/report_persistance.rb
index c85ad68..78ae47b 100755
--- a/report/report_persistance.rb
+++ b/report/report_persistance.rb
@@ -250,6 +250,7 @@ module Reports
     end
     
     def list_reports(type, filter_params={})
+      filter_params[:report_type] = type
       LOGGER.debug "find reports for params: "+filter_params.inspect
       reports = Lib::OhmUtil.find( ReportData, filter_params )
       reports.collect{ |r| r.id }
diff --git a/report/report_service.rb b/report/report_service.rb
index 722c3d6..3e23889 100644
--- a/report/report_service.rb
+++ b/report/report_service.rb
@@ -60,7 +60,7 @@ module Reports
     # call-seq:
     #   create_report(type, validation_uris) => string
     # 
-    def create_report(type, validation_uris, subjectid=nil, task=nil)
+    def create_report(type, validation_uris, identifier=nil, subjectid=nil, task=nil)
       
       LOGGER.info "create report of type '"+type.to_s+"'"
       check_report_type(type)
@@ -68,7 +68,10 @@ module Reports
       # step1: load validations
       raise OpenTox::BadRequestError.new("validation_uris missing") unless validation_uris
       LOGGER.debug "validation_uri(s): '"+validation_uris.inspect+"'"
-      validation_set = Reports::ValidationSet.new(validation_uris, subjectid)
+      LOGGER.debug "identifier: '"+identifier.inspect+"'"
+      raise "illegal num identifiers: "+identifier.size.to_s+" should be equal to num validation-uris ("+validation_uris.size.to_s+")" if
+        identifier and identifier.size!=validation_uris.size
+      validation_set = Reports::ValidationSet.new(validation_uris, identifier, subjectid)
       raise OpenTox::BadRequestError.new("cannot get validations from validation_uris '"+validation_uris.inspect+"'") unless validation_set and validation_set.size > 0
       LOGGER.debug "loaded "+validation_set.size.to_s+" validation/s"
       task.progress(10) if task
diff --git a/report/statistical_test.rb b/report/statistical_test.rb
index c37827e..1e586e2 100644
--- a/report/statistical_test.rb
+++ b/report/statistical_test.rb
@@ -58,7 +58,7 @@ module Reports
           end
         end
       end
-      {:titles => titles, :matrix => matrix}
+      {:titles => titles, :matrix => matrix, :num_results => grouped_validations[0].size}
     end
     
     def self.paired_ttest( validations1, validations2, attribute, significance_level=0.95 )
diff --git a/report/validation_access.rb b/report/validation_access.rb
index d0c3a1d..ffb7461 100755
--- a/report/validation_access.rb
+++ b/report/validation_access.rb
@@ -7,8 +7,9 @@ require "lib/validation_db.rb"
 #  
 class Reports::ValidationDB
   
-  def resolve_cv_uris(validation_uris, subjectid=nil)
-    res = []
+  def resolve_cv_uris(validation_uris, identifier=nil, subjectid=nil)
+    res = {}
+    count = 0
     validation_uris.each do |u|
       if u.to_s =~ /.*\/crossvalidation\/[0-9]+/
         cv_id = u.split("/")[-1].to_i
@@ -25,10 +26,13 @@ class Reports::ValidationDB
         raise OpenTox::NotFoundError.new "crossvalidation with id "+cv_id.to_s+" not found" unless cv
         raise OpenTox::BadRequestError.new("crossvalidation with id '"+cv_id.to_s+"' not finished") unless cv.finished
         #res += Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s}
-        res += Validation::Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|v| v.validation_uri.to_s }
+        Validation::Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).each do |v|
+          res[v.validation_uri.to_s] = identifier ? identifier[count] : nil
+        end
       else
-        res += [u.to_s]
+        res[u.to_s] = identifier ? identifier[count] : nil
       end
+      count += 1
     end
     res
   end
diff --git a/report/validation_data.rb b/report/validation_data.rb
index 11fa737..0c15dd2 100755
--- a/report/validation_data.rb
+++ b/report/validation_data.rb
@@ -1,6 +1,7 @@
 
 # the variance is computed when merging results for these attributes 
-VAL_ATTR_VARIANCE = [ :area_under_roc, :percent_correct, :root_mean_squared_error, :mean_absolute_error, :r_square, :accuracy  ]
+VAL_ATTR_VARIANCE = [ :area_under_roc, :percent_correct, :root_mean_squared_error, :mean_absolute_error, 
+  :r_square, :accuracy, :weighted_area_under_roc, :weighted_accuracy  ]
 VAL_ATTR_RANKING = [ :area_under_roc, :percent_correct, :true_positive_rate, :true_negative_rate, :weighted_area_under_roc ] #:accuracy ]
 
 ATTR_NICE_NAME = {}
@@ -51,21 +52,31 @@ end
 
 module Reports
   
+  @@validation_access = ValidationDB.new
+  @@persistance = ReportService.persistance
+  
+  def self.persistance
+    @@persistance
+  end
+  
+  def self.validation_access
+    @@validation_access
+  end
+  
+  # for overwriting validation source (other than using webservices)
+  def self.reset_validation_access(validation_access)
+    @@validation_access = validation_access
+  end
+  
+  
   # = ReportValidation
   #
   # contains all values of a validation object
   #
   class ReportValidation
     
-    @@validation_access = ValidationDB.new
-    
-    # for overwriting validation source (other than using webservices)
-    def self.reset_validation_access(validation_access)
-      @@validation_access = validation_access
-    end
-    
-    def self.resolve_cv_uris(validation_uris, subjectid)
-      @@validation_access.resolve_cv_uris(validation_uris, subjectid)
+    def self.resolve_cv_uris(validation_uris, identifier, subjectid)
+      Reports.validation_access.resolve_cv_uris(validation_uris, identifier, subjectid)
     end
     
     # create member variables for all validation properties
@@ -75,16 +86,17 @@ module Reports
     @@validation_attributes.each{ |a| attr_accessor a } 
   
     attr_reader :predictions
+    attr_accessor :identifier, :validation_report_uri, :crossvalidation_report_uri
     
     def initialize(uri = nil, subjectid = nil)
-      @@validation_access.init_validation(self, uri, subjectid) if uri
+      Reports.validation_access.init_validation(self, uri, subjectid) if uri
       @subjectid = subjectid
       #raise "subjectid is nil" unless subjectid
     end
     
     def self.from_cv_statistics( cv_uri, subjectid = nil )
       v = ReportValidation.new(nil, subjectid)
-      @@validation_access.init_validation_from_cv_statistics(v, cv_uri, subjectid)
+      Reports.validation_access.init_validation_from_cv_statistics(v, cv_uri, subjectid)
       v
     end
   
@@ -103,7 +115,7 @@ module Reports
           task.progress(100) if task
           nil
         else
-          @predictions = @@validation_access.get_predictions( self, @subjectid, task )
+          @predictions = Reports.validation_access.get_predictions( self, @subjectid, task )
         end
       end
     end
@@ -111,7 +123,7 @@ module Reports
     # returns the predictions feature values (i.e. the domain of the class attribute)
     #
     def get_accept_values()
-      @accept_values = @@validation_access.get_accept_values(self, @subjectid) unless @accept_values
+      @accept_values = Reports.validation_access.get_accept_values(self, @subjectid) unless @accept_values
       @accept_values
     end
     
@@ -119,36 +131,21 @@ module Reports
     #
     def feature_type
       return @feature_type if @feature_type!=nil
-      @feature_type = @@validation_access.feature_type(self, @subjectid) 
+      @feature_type = Reports.validation_access.feature_type(self, @subjectid) 
     end
     
     def predicted_variable
       return @predicted_variable if @predicted_variable!=nil
-      @predicted_variable = @@validation_access.predicted_variable(self, @subjectid) 
+      @predicted_variable = Reports.validation_access.predicted_variable(self, @subjectid) 
     end
     
     # loads all crossvalidation attributes, of the corresponding cv into this object 
     def load_cv_attributes
       raise "crossvalidation-id not set" unless @crossvalidation_id
-      @@validation_access.init_cv(self)
-    end
-    
-    @@persistance = ReportService.persistance
-    
-    def validation_report_uri
-      #puts "searching for validation report: "+self.validation_uri.to_s
-      return @validation_report_uri if @validation_report_uri!=nil
-      ids = @@persistance.list_reports("validation",{:validation_uris=>validation_uri })
-      @validation_report_uri = ReportService.instance.get_uri("validation",ids[-1]) if ids and ids.size>0
-    end
-    
-    def cv_report_uri
-      #puts "searching for cv report: "+self.crossvalidation_uri.to_s
-      return @cv_report_uri if @cv_report_uri!=nil
-      raise "no cv uri "+to_yaml unless self.crossvalidation_uri
-      ids = @@persistance.list_reports("crossvalidation",{:crossvalidation=>self.crossvalidation_uri.to_s })
-      #puts "-> "+ids.inspect
-      @cv_report_uri = ReportService.instance.get_uri("crossvalidation",ids[-1]) if ids and ids.size>0
+      Reports.validation_access.init_cv(self)
+      # load cv report
+      ids = Reports.persistance.list_reports("crossvalidation",{:crossvalidation=>self.crossvalidation_uri.to_s })
+      @crossvalidation_report_uri = ReportService.instance.get_uri("crossvalidation",ids[-1]) if ids and ids.size>0
     end
     
     def clone_validation
@@ -164,13 +161,20 @@ module Reports
   #
   class ValidationSet
     
-    def initialize(validation_uris=nil, subjectid=nil)
+    def initialize(validation_uris=nil, identifier=nil, subjectid=nil)
       @unique_values = {}
-      validation_uris = ReportValidation.resolve_cv_uris(validation_uris, subjectid) if validation_uris
-      @validations = Array.new
-      validation_uris.each{|u| @validations.push(ReportValidation.new(u, subjectid))} if validation_uris
+      @validations = []
+      if validation_uris    
+        validation_uri_and_ids = ReportValidation.resolve_cv_uris(validation_uris, identifier, subjectid)
+        validation_uri_and_ids.each do |u,id|
+          v = ReportValidation.new(u, subjectid)
+          v.identifier = id if id
+          ids = Reports.persistance.list_reports("validation",{:validation_uris=>v.validation_uri })
+          v.validation_report_uri = ReportService.instance.get_uri("validation",ids[-1]) if ids and ids.size>0
+          @validations << v
+        end
+      end
     end
-
   
     def self.create(validations)
       set = ValidationSet.new
@@ -399,6 +403,17 @@ module Reports
       return array
     end
     
+    def replace_with_cv_stats
+      new_set = ValidationSet.new
+      grouping = Util.group(@validations, [:crossvalidation_id])
+      grouping.each do |g|
+        v = ReportValidation.from_cv_statistics(g[0].crossvalidation_uri)
+        v.identifier = g.collect{|vv| vv.identifier}.uniq.join(";")
+        new_set.validations << v 
+      end
+      return new_set
+    end
+    
     # creates a new validaiton set, that contains merged validations
     # all validation with equal values for __equal_attributes__ are summed up in one validation, i.e. merged 
     #
@@ -416,31 +431,16 @@ module Reports
       grouping = Util.group(@validations, equal_attributes)
       #puts "groups "+grouping.size.to_s
 
-      if ( equal_attributes.include?(:crossvalidation_id) )
-        # do not merge, use crossvalidation statistics
-        raise "statistics vs merging problem" if equal_attributes.size!=1
-        grouping.each do |g|
-          new_set.validations << ReportValidation.from_cv_statistics(g[0].crossvalidation_uri)
-        end
-      else
-        #merge
-        Lib::MergeObjects.register_merge_attributes( ReportValidation,
-          Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL) unless 
-            Lib::MergeObjects.merge_attributes_registered?(ReportValidation)
-        grouping.each do |g|
-          new_set.validations << g[0].clone_validation
-          w = 1
-          g[1..-1].each do |v|
-            new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v,w,1)
-            w+=1
-          end
+      #merge
+      Lib::MergeObjects.register_merge_attributes( ReportValidation,
+        Validation::VAL_MERGE_AVG+Validation::VAL_MERGE_SUM,[],Validation::VAL_MERGE_GENERAL+[:identifier, :validation_report_uri, :crossvalidation_report_uri]) unless 
+          Lib::MergeObjects.merge_attributes_registered?(ReportValidation)
+      grouping.each do |g|
+        new_set.validations << g[0].clone_validation
+        g[1..-1].each do |v|
+          new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v)
         end
       end
-      
-      new_set.validations.each do |v|
-        raise "not a validation "+v.class.to_s+" "+v.to_s unless v.is_a?(Reports::ReportValidation)
-      end
-      
       return new_set
     end
     
diff --git a/report/xml_report.rb b/report/xml_report.rb
index 4fbfae3..7467c47 100755
--- a/report/xml_report.rb
+++ b/report/xml_report.rb
@@ -187,7 +187,13 @@ module Reports
           if auto_link_urls && v.to_s =~ /depict/ || v.to_s =~ /image\/png$/ #PENDING 
             add_image(entry, v.to_s)
           elsif auto_link_urls && v.to_s =~ /^http(s?):\/\//
-           add_url(entry, v.to_s, v.to_s)
+           #add_url(entry, v.to_s, v.to_s)
+           v.to_s.split(" ").each do |vv|
+              add_url(entry, vv.to_s, vv.to_s)
+              space = Element.new("para")
+              space.text = " "
+              entry << space
+           end
           else
            entry.text = v.to_s
           end
@@ -221,11 +227,15 @@ module Reports
       return list
     end
     
-    def add_url (element, url, description=url )
-      
+    def url_element( url, description=url )
       ulink = Element.new("ulink")
       ulink.add_attributes({"url" => url})
       ulink.text = description
+      ulink
+    end
+    
+    def add_url (element, url, description=url )
+      ulink = url_element(url, description)
       element << ulink
       return ulink
     end
diff --git a/test/test_examples.rb b/test/test_examples.rb
index 49d7838..eb0543f 100755
--- a/test/test_examples.rb
+++ b/test/test_examples.rb
@@ -74,6 +74,14 @@ module ValidationExamples
     end
   end
   
+  class LazarLastEPAFHMSplit < LazarEPAFHMSplit
+    def initialize
+      super
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/last")
+    end
+  end
+
+  
   class MajorityEPAFHMSplit < EPAFHMSplit
     def initialize
       @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm")
@@ -81,11 +89,18 @@ module ValidationExamples
     end
   end
   
+    class MajorityRandomEPAFHMSplit < MajorityEPAFHMSplit
+    def initialize
+      @algorithm_params = "random=true"
+      super
+    end
+  end
+  
     ########################################################################################################
   
     class EPAFHMCrossvalidation < CrossValidation
     def initialize
-      @dataset_file = File.new("data/EPAFHM.mini.csv","r")
+      @dataset_file = File.new("data/EPAFHM.med.csv","r")
       #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk"
       @num_folds = 10
     end
@@ -93,11 +108,20 @@ module ValidationExamples
   
   class MajorityEPAFHMCrossvalidation < EPAFHMCrossvalidation
     def initialize
+      #@dataset_uri = "http://local-ot/dataset/2366"
+      #@prediction_feature = "http://local-ot/dataset/2366/feature/LC50_mmol"
       @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm")
       super
     end
   end
 
+  class MajorityRandomEPAFHMCrossvalidation < MajorityEPAFHMCrossvalidation
+    def initialize
+      @algorithm_params = "random=true"
+      super
+    end
+  end
+
   class LazarEPAFHMCrossvalidation < EPAFHMCrossvalidation
     def initialize
       @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
@@ -125,6 +149,14 @@ module ValidationExamples
       super
     end
   end
+
+  class LazarLastHamsterSplit < LazarHamsterSplit
+    def initialize
+      super
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/last")
+    end
+  end
+
   
   class MajorityHamsterSplit < HamsterSplit
     def initialize
@@ -133,6 +165,13 @@ module ValidationExamples
     end
   end
   
+    class MajorityRandomHamsterSplit < MajorityHamsterSplit
+    def initialize
+      @algorithm_params = "random=true"
+      super
+    end
+  end
+  
   ########################################################################################################
   
   class HamsterBootstrapping < BootstrappingValidation
@@ -212,6 +251,13 @@ module ValidationExamples
       super
     end
   end
+  
+    class MajorityRandomHamsterCrossvalidation < MajorityHamsterCrossvalidation
+    def initialize
+      @algorithm_params = "random=true"
+      super
+    end
+  end
 
   class LazarHamsterCrossvalidation < HamsterCrossvalidation
     def initialize
@@ -221,6 +267,45 @@ module ValidationExamples
     end
   end
   
+  class LazarLastHamsterCrossvalidation < LazarHamsterCrossvalidation
+    def initialize
+      super
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/last")
+    end
+  end
+  
+  ########################################################################################################
+  
+  class LazarHamsterMiniCrossvalidation < CrossValidation
+    def initialize
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
+      @dataset_file = File.new("data/hamster_carcinogenicity.mini.csv","r")
+      @num_folds = 2
+    end
+  end  
+  
+  class ISSCANStratifiedCrossvalidation < CrossValidation
+    def initialize
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
+      @dataset_file = File.new("data/ISSCAN_v3a_canc-red.csv","r")
+      @stratified = true
+      @num_folds = 10
+    end
+  end  
+  
+  class ISSCAN2StratifiedCrossvalidation < CrossValidation
+    def initialize
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
+      @dataset_file = File.new("data/ISSCAN_v3a_sal.csv","r")
+      @stratified = true
+      @num_folds = 10
+    end
+  end  
+  
+  
   ########################################################################################################  
 
   class ISTHamsterCrossvalidation < CrossValidation
@@ -368,6 +453,74 @@ module ValidationExamples
     end
   end
   
+  class NtuaModel2 < ModelValidation
+    def initialize
+      @model_uri = "http://opentox.ntua.gr:8080/model/11093fbc-3b8b-41e2-bfe3-d83f5f529efc"
+      @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/54"
+      @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/579820"
+    end
+  end
+  
+  class NtuaModel3 < ModelValidation
+    def initialize
+      @model_uri = "http://opentox.ntua.gr:8080/model/bbab3714-e90b-4990-bef9-8e7d3a30eece"
+      @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+      #@prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/579820"
+    end
+  end  
+  
+  ########################################################################################################
+  
+  class NtuaTrainingTest < TrainingTestValidation
+    def initialize
+      @algorithm_uri = "http://opentox.ntua.gr:8080/algorithm/mlr"
+      @training_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+      @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+      @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200"
+    end
+  end  
+
+  class NtuaTrainingTestSplit < SplitTestValidation
+    def initialize
+      @algorithm_uri = "http://opentox.ntua.gr:8080/algorithm/mlr"
+      @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+      @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200"
+    end
+  end  
+  
+    class NtuaCrossvalidation < CrossValidation
+    def initialize
+      @algorithm_uri = "http://opentox.ntua.gr:8080/algorithm/mlr"
+      @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+      @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200"
+    end
+  end  
+  
+  class AmbitVsNtuaTrainingTest < TrainingTestValidation
+    def initialize
+      @algorithm_uri = "http://apps.ideaconsult.net:8080/ambit2/algorithm/LR"
+      @training_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+      @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+      @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200"
+    end
+  end  
+  
+  
+  class LazarVsNtuaCrossvalidation < CrossValidation
+    def initialize
+      @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar")
+      @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc")
+      @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+      @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200"
+      @num_folds=3
+    end
+  end  
+
+
+#  loading prediciton via test-dataset:'http://apps.ideaconsult.net:8080/ambit2/dataset/R545', 
+#  test-target-datset:'', prediction-dataset:'http://apps.ideaconsult.net:8080/ambit2/dataset/584389', 
+#  prediction_feature: 'http://apps.ideaconsult.net:8080/ambit2/feature/22200' ', predicted_variable: 'http://apps.ideaconsult.net:8080/ambit2/feature/627667'           :: /ot_predictions.rb:21:in `initialize'
+#D, [2011-05-11T13:47:26.631628 #22952] DEBUG -- : validation         :: 
   ########################################################################################################
   
   class TumModel < ModelValidation
@@ -402,6 +555,23 @@ module ValidationExamples
     end
   end  
   
+  class AmbitXYModelValidation < ModelValidation
+    def initialize
+      @model_uri =  "http://apps.ideaconsult.net:8080/ambit2/model/237692"
+      @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R736156"
+      @prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/430905"
+    end
+  end 
+  
+    class AmbitXYZModelValidation < ModelValidation
+    def initialize
+      @model_uri =  "http://apps.ideaconsult.net:8080/ambit2/model/238008"
+      @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R736396"
+      #@prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/430905" ??
+    end
+  end 
+  
+  
   class AmbitTrainingTest < TrainingTestValidation
     def initialize
       @training_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401560"
@@ -528,20 +698,36 @@ module ValidationExamples
     end
   end  
   
+      ########################################################################################################
+  
+  class TumCrossValidation < CrossValidation
+    def initialize
+      @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/124963"
+      @algorithm_uri = "http://opentox:8080/OpenTox/algorithm/kNNregression"
+      @prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/121905"
+      @num_folds=2
+      super
+    end
+  end
+  
    ########################################################################################################
   
   @@list = {
-      "1" => [ LazarHamsterSplit, MajorityHamsterSplit ],
+      "1" => [ LazarHamsterSplit, MajorityHamsterSplit, MajorityRandomHamsterSplit ],
       "1a" => [ LazarHamsterSplit ],
       "1b" => [ MajorityHamsterSplit ],
+      "1c" => [ MajorityRandomHamsterSplit ],
+      "1d" => [ LazarLastHamsterSplit ],
       
       "2" => [ LazarHamsterTrainingTest, MajorityHamsterTrainingTest ],
       "2a" => [ LazarHamsterTrainingTest ],
       "2b" => [ MajorityHamsterTrainingTest ],
       
-      "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation ],
+      "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation, MajorityRandomHamsterCrossvalidation ],
       "3a" => [ LazarHamsterCrossvalidation ],
       "3b" => [ MajorityHamsterCrossvalidation ],
+      "3c" => [ MajorityRandomHamsterCrossvalidation ],
+      "3d" => [ LazarLastHamsterCrossvalidation ],
       
       "4" => [ MajorityISTHamsterCrossvalidation, LazarISTHamsterCrossvalidation, ISTLazarISTHamsterCrossvalidation ],
       "4a" => [ MajorityISTHamsterCrossvalidation ],
@@ -574,11 +760,17 @@ module ValidationExamples
       
       "13a" =>  [ LazarEPAFHMSplit ],
       "13b" =>  [ MajorityEPAFHMSplit ],
+      "13c" =>  [ MajorityRandomEPAFHMSplit ],
+      "13d" =>  [ LazarLastEPAFHMSplit ],
       
+      "14" =>   [ LazarEPAFHMCrossvalidation, MajorityEPAFHMCrossvalidation, MajorityRandomEPAFHMCrossvalidation ],
       "14a" =>  [ LazarEPAFHMCrossvalidation ],
       "14b" =>  [ MajorityEPAFHMCrossvalidation ],
+      "14c" =>  [ MajorityRandomEPAFHMCrossvalidation ],
       
       "15a" =>  [ NtuaModel ],
+      "15b" =>  [ NtuaModel2 ],
+      "15c" =>  [ NtuaModel3 ],
       
       "16" => [ LazarRepdoseSplit, MajorityRepdoseSplit ],
       "16a" => [ LazarRepdoseSplit ],
@@ -599,7 +791,19 @@ module ValidationExamples
       "19g" => [ AmbitJ48TrainingTest ],
       "19h" => [ AmbitJ48TrainingTestSplit ],
       "19i" => [ AmbitAquaticModelValidation ],
+      "19j" => [ AmbitXYModelValidation ],
+      
+      "20a" => [ TumCrossValidation ],
+      
+      "21a" => [ LazarHamsterMiniCrossvalidation ],
+      "21b" => [ ISSCANStratifiedCrossvalidation ],
+      "21c" => [ ISSCAN2StratifiedCrossvalidation ],
       
+      "22a" =>  [ NtuaTrainingTest ],
+      "22b" =>  [ NtuaTrainingTestSplit ],
+      "22c" =>  [ NtuaCrossvalidation ],
+      "22d" =>  [ LazarVsNtuaCrossvalidation ],
+      "22e" =>  [ AmbitVsNtuaTrainingTest ],
       
     }
   
diff --git a/validation/validation_service.rb b/validation/validation_service.rb
index 99d8672..520d84c 100755
--- a/validation/validation_service.rb
+++ b/validation/validation_service.rb
@@ -63,13 +63,13 @@ module Validation
                :percent_unpredicted => prediction.percent_unpredicted,
                :finished => true
         (VAL_PROPS_GENERAL-[:validation_uri]).each do |p|
-          v.send("#{p.to_s}=".to_sym, vals.collect{ |vv| vv.send(p) }.uniq.join(","))
+          v.send("#{p.to_s}=".to_sym, vals.collect{ |vv| vv.send(p) }.uniq.join(";"))
         end
         v.date = crossvalidation.date
         v.validation_type = "crossvalidation_statistics"
         v.crossvalidation_id = crossvalidation.id
-        v.crossvalidation_fold = vals.collect{ |vv| vv.crossvalidation_fold }.uniq.join(",")       
-        v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(",")
+        v.crossvalidation_fold = vals.collect{ |vv| vv.crossvalidation_fold }.uniq.join(";")       
+        v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(";")
         v.save
       end
       v
diff --git a/validation/validation_test.rb b/validation/validation_test.rb
index efa8ad5..2c86548 100755
--- a/validation/validation_test.rb
+++ b/validation/validation_test.rb
@@ -8,7 +8,7 @@ before {
 
 require "uri"
 require "yaml"
-ENV['RACK_ENV'] = 'test'
+ENV['RACK_ENV'] = 'production'
 require 'application.rb'
 require 'test/unit'
 require 'rack/test'
@@ -20,10 +20,10 @@ LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S "
 LOGGER.formatter = Logger::Formatter.new
 
 if AA_SERVER
-  TEST_USER = "mgtest"
-  TEST_PW = "mgpasswd"
-  #TEST_USER = "guest"
-  #TEST_PW = "guest"
+  #TEST_USER = "mgtest"
+  #TEST_PW = "mgpasswd"
+  TEST_USER = "guest"
+  TEST_PW = "guest"
   SUBJECTID = OpenTox::Authorization.authenticate(TEST_USER,TEST_PW)
   raise "could not log in" unless SUBJECTID
   puts "logged in: "+SUBJECTID.to_s
@@ -60,6 +60,171 @@ class ValidationTest < Test::Unit::TestCase
     begin
       $test_case = self
       
+      #get 'crossvalidation/138/statistics'
+      #get 'crossvalidation/189/statistics'
+      #puts last_response.body
+#      run_test("1b")       
+
+      #get '/crossvalidation/79/predictions',nil,'HTTP_ACCEPT' => "application/x-yaml"
+      #puts last_response.body
+      
+      #run_test("22e") #,:validation_uri=>"http://local-ot/validation/84" )
+      #run_test("21b")
+      #run_test("21c")
+
+     # get '?media=text/uri-list'      
+
+      #post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/135,http://local-ot/validation/crossvalidation/134"}
+      #post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/174,http://local-ot/validation/crossvalidation/175"}
+      # 2 majority, 175 is real maj, 176 is random
+      
+#      post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/185,http://local-ot/validation/crossvalidation/193,http://local-ot/validation/crossvalidation/186,http://local-ot/validation/crossvalidation/194,http://local-ot/validation/crossvalidation/187,http://local-ot/validation/crossvalidation/195",
+#        :identifier=>"lazar,lazar,real_majority,real_majority,random_classification,random_classification"}
+#      uri = last_response.body
+#      rep = wait_for_task(uri)
+#      puts rep
+
+#      post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/199,http://local-ot/validation/crossvalidation/204,http://local-ot/validation/crossvalidation/203",
+#        :identifier=>"lazar,real_majority,random_classification"}
+#      uri = last_response.body
+#      rep = wait_for_task(uri)
+#      puts rep
+      # 205 206 207
+
+#      post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/149,http://local-ot/validation/crossvalidation/210",
+#        :identifier=>"bbrc,last"}
+#      uri = last_response.body
+#      rep = wait_for_task(uri)
+#      puts rep
+      
+      #run_test("1a", {:validation_uri=>"http://local-ot/validation/466"})
+#      puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+      #run_test("3a",{:validation_uri=>"http://local-ot/validation/crossvalidation/149"})
+      #puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+      #run_test("13a", {:validation_uri=>"http://local-ot/validation/406"})
+#      puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+      #run_test("14a",{:validation_uri=>"http://local-ot/validation/crossvalidation/148"})
+#      puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+      run_test("1a")
+      
+#      run_test("3d",{
+#        :dataset_uri => "http://local-ot/dataset/2897", 
+#        :prediction_feature => "http://local-ot/dataset/2897/feature/Hamster%20Carcinogenicity",
+#        :random_seed => 1
+#        })
+
+      #run_test("14",{
+      #  :dataset_uri => "http://local-ot/dataset/3877", 
+      #  :prediction_feature => "http://local-ot/dataset/3877/feature/LC50_mmol",
+      #  :random_seed => 2
+      #  })
+      #puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+#      get "?model=http://local-ot/model/330"
+#      puts last_response.body
+#      puts "\n\n"
+#      get ""
+#      puts last_response.body
+
+      #get "report/validation?validation=http://local-ot/validation/167"
+      #puts last_response.body
+
+#      run_test("3a") #,:validation_uri=>"http://local-ot/validation/84" )
+      #get "report/crossvalidation?crossvalidation=http://local-ot/validation/crossvalidation/47"
+      #puts last_response.body
+      
+  
+    rescue => ex
+      rep = OpenTox::ErrorReport.create(ex, "")
+      puts rep.to_yaml
+    ensure
+      #OpenTox::Authorization.logout(SUBJECTID) if AA_SERVER
+    end
+  end
+
+  def app
+    Sinatra::Application
+  end
+  
+  def run_test(select=nil, overwrite={}, delete=false )
+    
+    if AA_SERVER && SUBJECTID && delete
+      policies_before = OpenTox::Authorization.list_policy_uris(SUBJECTID)
+    end
+    
+    puts ValidationExamples.list unless select
+    validationExamples = ValidationExamples.select(select)
+    validationExamples.each do |vv|
+      vv.each do |v| 
+        ex = v.new
+        ex.subjectid = SUBJECTID
+        
+        overwrite.each do |k,v|
+          ex.send(k.to_s+"=",v)
+        end
+        
+        unless ex.validation_uri
+          ex.upload_files
+          ex.check_requirements
+          ex.validate
+            
+          LOGGER.debug "validation done '"+ex.validation_uri.to_s+"'"
+        end
+        if !delete and ex.validation_uri
+          if SUBJECTID
+            puts ex.validation_uri+"?subjectid="+CGI.escape(SUBJECTID)
+          else
+            puts ex.validation_uri
+          end
+        end
+          
+        unless ex.report_uri
+          ex.report
+        end
+        if !delete and ex.report_uri
+          if SUBJECTID  
+            puts ex.report_uri+"?subjectid="+CGI.escape(SUBJECTID)
+          else
+            puts ex.report_uri
+          end
+        end
+        ##ex.verify_yaml
+        ##ex.compare_yaml_vs_rdf
+        ex.delete if delete
+      end
+    end
+    
+    if AA_SERVER && SUBJECTID && delete
+      policies_after= OpenTox::Authorization.list_policy_uris(SUBJECTID)
+      diff = policies_after.size - policies_before.size
+      if (diff != 0)
+        policies_before.each do |k,v|
+          policies_after.delete(k)
+        end
+        LOGGER.warn diff.to_s+" policies NOT deleted:\n"+policies_after.collect{|k,v| k.to_s+" => "+v.to_s}.join("\n")
+      else
+        LOGGER.debug "all policies deleted"
+      end
+    end      
+  end
+  
+  def prepare_examples
+    get '/prepare_examples'
+  end  
+  
+ def do_test_examples # USES CURL, DO NOT FORGET TO RESTART
+   post '/test_examples'
+ end
+ 
+  def do_test_examples_ortona 
+   post '/test_examples',:examples=>"http://ortona.informatik.uni-freiburg.de/validation/examples"
+ end
+  
+end
+
+
+
 #    prediction_feature = "https://ambit.uni-plovdiv.bg:8443/ambit2/feature/26221"
 #    puts OpenTox::Feature.find(prediction_feature).domain.inspect
 #      exit
@@ -137,6 +302,18 @@ class ValidationTest < Test::Unit::TestCase
   #      :regression=>"true"}
   #      #:classification=>"true"}
   #    puts last_response.body
+
+#      post "/validate_datasets",{
+#        :test_dataset_uri=>"http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=10",
+#        :prediction_dataset_uri=>"http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=10",
+#        #:test_target_dataset_uri=>"http://local-ot/dataset/202",
+#        :prediction_feature=>"http://apps.ideaconsult.net:8080/ambit2/feature/21573",
+#        :predicted_feature=>"http://apps.ideaconsult.net:8080/ambit2/feature/21573",
+#        #:regression=>"true"}
+#        :classification=>"true"}
+#      puts last_response.body
+
+     #run_test("1a") #,:validation_uri=>"http://local-ot/validation/84" )
   
   #     post "/validate_datasets",{
   #      :test_dataset_uri=>"http://local-ot/dataset/89",
@@ -251,13 +428,17 @@ class ValidationTest < Test::Unit::TestCase
       #puts ""
       #puts last_response.body 
       #exit
+
+#      run_test("20a")
       
 #      get "/error"
 #      puts last_response.body
 
       #delete "/1",:subjectid=>SUBJECTID
       
-      prepare_examples()
+      #prepare_examples()
+
+      #run_test("15b")
       
       #run_test("1a") #,{:validation_uri => "http://local-ot/validation/crossvalidation/1"})
       
@@ -275,92 +456,4 @@ class ValidationTest < Test::Unit::TestCase
       
       #prepare_examples
       #do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE
-      #do_test_examples_ortona
-    
-    rescue => ex
-      rep = OpenTox::ErrorReport.create(ex, "")
-      puts rep.to_yaml
-    ensure
-      #OpenTox::Authorization.logout(SUBJECTID) if AA_SERVER
-    end
-  end
-
-  def app
-    Sinatra::Application
-  end
-  
-  def run_test(select=nil, overwrite={}, delete=false )
-    
-    if AA_SERVER && SUBJECTID && delete
-      policies_before = OpenTox::Authorization.list_policy_uris(SUBJECTID)
-    end
-    
-    puts ValidationExamples.list unless select
-    validationExamples = ValidationExamples.select(select)
-    validationExamples.each do |vv|
-      vv.each do |v| 
-        ex = v.new
-        ex.subjectid = SUBJECTID
-        
-        overwrite.each do |k,v|
-          ex.send(k.to_s+"=",v)
-        end
-        
-        unless ex.validation_uri
-          ex.upload_files
-          ex.check_requirements
-          ex.validate
-            
-          LOGGER.debug "validation done '"+ex.validation_uri.to_s+"'"
-        end
-        if !delete and ex.validation_uri
-          if SUBJECTID
-            puts ex.validation_uri+"?subjectid="+CGI.escape(SUBJECTID)
-          else
-            puts ex.validation_uri
-          end
-        end
-          
-        unless ex.report_uri
-          ex.report
-        end
-        if !delete and ex.report_uri
-          if SUBJECTID  
-            puts ex.report_uri+"?subjectid="+CGI.escape(SUBJECTID)
-          else
-            puts ex.report_uri
-          end
-        end
-        ##ex.verify_yaml
-        ##ex.compare_yaml_vs_rdf
-        ex.delete if delete
-      end
-    end
-    
-    if AA_SERVER && SUBJECTID && delete
-      policies_after= OpenTox::Authorization.list_policy_uris(SUBJECTID)
-      diff = policies_after.size - policies_before.size
-      if (diff != 0)
-        policies_before.each do |k,v|
-          policies_after.delete(k)
-        end
-        LOGGER.warn diff.to_s+" policies NOT deleted:\n"+policies_after.collect{|k,v| k.to_s+" => "+v.to_s}.join("\n")
-      else
-        LOGGER.debug "all policies deleted"
-      end
-    end      
-  end
-  
-  def prepare_examples
-    get '/prepare_examples'
-  end  
-  
- def do_test_examples # USES CURL, DO NOT FORGET TO RESTART
-   post '/test_examples'
- end
- 
-  def do_test_examples_ortona 
-   post '/test_examples',:examples=>"http://ortona.informatik.uni-freiburg.de/validation/examples"
- end
-  
-end
+      #do_test_examples_ortona
\ No newline at end of file