From ee0eb28550ada392b7903a49d650f40c695e8612 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Sat, 9 Jun 2012 10:36:27 +0200 Subject: new branch val_exp --- .gitignore | 1 + lib/dataset_cache.rb | 1 + lib/dataset_util.rb | 0 lib/prediction_data.rb | 34 ++++++++++++----- lib/validation_db.rb | 1 + report/plot_factory.rb | 6 ++- report/report_content.rb | 22 +++++------ report/report_factory.rb | 23 ++++++++---- report/statistical_test.rb | 14 +++++-- report/validation_access.rb | 49 +++++++++++++++++------- report/validation_data.rb | 1 + validation/validation_application.rb | 72 ++++++++++++++++++++++++++++++++++-- validation/validation_service.rb | 8 ++-- validation/validation_test.rb | 28 +++++++++----- 14 files changed, 198 insertions(+), 62 deletions(-) create mode 100644 lib/dataset_util.rb diff --git a/.gitignore b/.gitignore index 8e289af..c64786a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ reports public/server.log nightly/nightly_report.html nightly/nightly_report.xml +.buildpath diff --git a/lib/dataset_cache.rb b/lib/dataset_cache.rb index 1af1d51..7a13e9b 100644 --- a/lib/dataset_cache.rb +++ b/lib/dataset_cache.rb @@ -12,6 +12,7 @@ module Lib return nil if (dataset_uri==nil) d = @@cache[dataset_uri.to_s+"_"+subjectid.to_s] if d==nil + LOGGER.debug "loading dataset #{dataset_uri}" d = OpenTox::Dataset.find(dataset_uri, subjectid) @@cache[dataset_uri.to_s+"_"+subjectid.to_s] = d end diff --git a/lib/dataset_util.rb b/lib/dataset_util.rb new file mode 100644 index 0000000..e69de29 diff --git a/lib/prediction_data.rb b/lib/prediction_data.rb index d387d24..5f11957 100644 --- a/lib/prediction_data.rb +++ b/lib/prediction_data.rb @@ -172,11 +172,16 @@ module Lib predicted_values = [] confidence_values = [] + dup_compounds = [] + dup_actual_values = [] count = 0 - compounds.each do |c| + compounds.size.times do |i| + c = compounds[i] if prediction_dataset.compounds.index(c)==nil predicted_values << nil confidence_values << nil + dup_compounds << c + dup_actual_values << actual_values[i] else case feature_type when "classification" @@ -184,19 +189,28 @@ module Lib when "regression" vals = regression_vals(prediction_dataset, c, predicted_variable) end - raise "not yet implemented: more than one prediction for one compound" if vals.size>1 - predicted_values << vals[0] - if predicted_confidence - confidence_values << confidence_val(prediction_dataset, c, predicted_confidence) - else - confidence_values << nil - end + + #vals.uniq! #more than one prediciton is ok if it yields the equal value + #raise "not yet implemented: more than one prediction for one compound '#{vals.inspect}'" if vals.size>1 + #predicted_values << vals[0] + + vals.each do |val| + dup_compounds << c + dup_actual_values << actual_values[i] + predicted_values << val + + if predicted_confidence + confidence_values << confidence_val(prediction_dataset, c, predicted_confidence) + else + confidence_values << nil + end + end end count += 1 end - all_compounds += compounds + all_compounds += dup_compounds all_predicted_values += predicted_values - all_actual_values += actual_values + all_actual_values += dup_actual_values all_confidence_values += confidence_values task.progress( task_status += task_step ) if task # loaded predicted values and confidence diff --git a/lib/validation_db.rb b/lib/validation_db.rb index 086853e..17bd506 100755 --- a/lib/validation_db.rb +++ b/lib/validation_db.rb @@ -85,6 +85,7 @@ module Validation index :prediction_feature index :training_dataset_uri index :test_dataset_uri + index :finished attr_accessor :subjectid diff --git a/report/plot_factory.rb b/report/plot_factory.rb index 6e90dbc..61c3eea 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -246,7 +246,11 @@ module Reports data[v.send(title_attribute).to_s] << value end - Reports::r_util.boxplot( out_files, data) + data_array = [] + data.each do |k,v| + data_array << [k, v] + end + Reports::r_util.boxplot( out_files, data_array.sort) end def self.create_bar_plot( out_files, validation_set, title_attribute, value_attributes ) diff --git a/report/report_content.rb b/report/report_content.rb index 033b367..03adc30 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -28,11 +28,11 @@ class Reports::ReportContent end_section() end - def add_paired_ttest_tables( validation_set, + def add_ttest_tables( validation_set, group_attribute, test_attributes, ttest_level = 0.9, - section_title = "Paired t-test", + section_title = "T-Test", section_text = nil) raise "no test_attributes given: "+test_attributes.inspect unless test_attributes.is_a?(Array) and test_attributes.size>0 @@ -46,7 +46,7 @@ class Reports::ReportContent accept_values.each do |accept_value| test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, - group_attribute, test_attribute, accept_value, "paired_ttest", ttest_level ) + group_attribute, test_attribute, accept_value, "ttest", ttest_level ) #puts test_matrix.inspect titles = test_matrix[:titles] matrix = test_matrix[:matrix] @@ -356,7 +356,7 @@ class Reports::ReportContent @xml_report.add_paragraph(section_box, section_text) if section_text plot_png = nil; plot_svg = nil - begin + #begin plot_input = [] value_attributes.each do |a| accept = validation_set.get_accept_values_for_attr(a) @@ -388,13 +388,13 @@ class Reports::ReportContent @xml_report.add_imagefigures_in_row(section_box,f,"Boxplots #{i}") i+=1 end - rescue Exception => ex - msg = "WARNING could not create box plot: "+ex.message - LOGGER.error(msg) - rm_tmp_file(plot_png[:name]) if plot_png - rm_tmp_file(plot_svg[:name]) if plot_svg - @xml_report.add_paragraph(section_box, msg) - end + #rescue Exception => ex + # msg = "WARNING could not create box plot: "+ex.message + # LOGGER.error(msg) + # rm_tmp_file(plot_png[:name]) if plot_png + # rm_tmp_file(plot_svg[:name]) if plot_svg + # @xml_report.add_paragraph(section_box, msg) + #end end private diff --git a/report/report_factory.rb b/report/report_factory.rb index f73ffd9..b67fbf1 100755 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -14,7 +14,7 @@ VAL_ATTR_REGR = [ :num_instances, :num_unpredicted, :root_mean_squared_error, #VAL_ATTR_BOX_PLOT_CLASS = [ :accuracy, :average_area_under_roc, # :area_under_roc, :f_measure, :true_positive_rate, :true_negative_rate ] VAL_ATTR_BOX_PLOT_CLASS = [ :accuracy, :area_under_roc, :f_measure, :true_positive_rate, :true_negative_rate, :positive_predictive_value, :negative_predictive_value ] -VAL_ATTR_BOX_PLOT_REGR = [ :root_mean_squared_error, :mean_absolute_error, :r_square ] +VAL_ATTR_BOX_PLOT_REGR = [ :root_mean_squared_error, :mean_absolute_error, :r_square, :concordance_correlation_coefficient ] VAL_ATTR_TTEST_REGR = [ :r_square, :root_mean_squared_error ] VAL_ATTR_TTEST_CLASS = [ :accuracy, :average_area_under_roc ] @@ -61,7 +61,7 @@ module Reports::ReportFactory i = 0 task_step = 100 / validation_set.size.to_f validation_set.validations.each do |v| - v.get_predictions( OpenTox::SubTask.create(task, i*task_step, (i+1)*task_step ) ) + v.get_predictions( OpenTox::SubTask.create(task, i*task_step, [(i+1)*task_step,100].min ) ) i += 1 end end @@ -299,7 +299,7 @@ module Reports::ReportFactory report.add_result(merged,result_attributes,res_titel,res_titel,res_text) # pending: regression stats have different scales!!! report.add_box_plot(set, :identifier, box_plot_attributes) - report.add_paired_ttest_tables(set, :identifier, ttest_attributes, ttest_significance) if ttest_significance>0 + report.add_ttest_tables(set, :identifier, ttest_attributes, ttest_significance) if ttest_significance>0 report.end_section end task.progress(100) if task @@ -314,7 +314,7 @@ module Reports::ReportFactory validation_set.get_values(:identifier).inspect) if validation_set.num_different_values(:identifier)<2 #validation_set.load_cv_attributes - pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) ) + #pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) ) report = Reports::ReportContent.new("Method comparison report") add_filter_warning(report, validation_set.filter_params) if validation_set.filter_params!=nil @@ -322,24 +322,33 @@ module Reports::ReportFactory case validation_set.unique_feature_type when "classification" result_attributes += VAL_ATTR_CLASS + ttest_attributes = VAL_ATTR_TTEST_CLASS box_plot_attributes = VAL_ATTR_BOX_PLOT_CLASS else result_attributes += VAL_ATTR_REGR + ttest_attributes = VAL_ATTR_TTEST_REGR box_plot_attributes = VAL_ATTR_BOX_PLOT_REGR end merged = validation_set.merge([:identifier]) merged.sort(:identifier) - merged.validations.each do |v| v.validation_uri = v.validation_uri.split(";").uniq.join(" ") v.validation_report_uri = v.validation_report_uri.split(";").uniq.join(" ") if v.validation_report_uri end - msg = merged.validations.collect{|v| v.identifier+" ("+Lib::MergeObjects.merge_count(v).to_s+"x)"}.join(", ") report.add_result(merged,result_attributes,"Average Results","Results",msg) - report.add_box_plot(validation_set, :identifier, box_plot_attributes) + if params[:ttest_attributes] and params[:ttest_attributes].chomp.size>0 + ttest_attributes = params[:ttest_attributes].split(",").collect{|a| a.to_sym} + end + ttest_significance = 0.9 + if params[:ttest_significance] + ttest_significance = params[:ttest_significance].to_f + end + #report.add_ttest_tables(validation_set, :identifier, ttest_attributes, ttest_significance) if ttest_significance>0 + #report.add_ttest_tables(validation_set, :identifier, ttest_attributes, 0.75) if ttest_significance>0 + #report.add_ttest_tables(validation_set, :identifier, ttest_attributes, 0.5) if ttest_significance>0 report end diff --git a/report/statistical_test.rb b/report/statistical_test.rb index da46f6b..fd4b810 100644 --- a/report/statistical_test.rb +++ b/report/statistical_test.rb @@ -6,7 +6,7 @@ module Reports class ReportStatisticalTest # __grouped_validations__ : array of validation arrays - def self.test_matrix( validations, group_attribute, test_attribute, class_value, test_method="paired_ttest", significance_level=0.95 ) + def self.test_matrix( validations, group_attribute, test_attribute, class_value, test_method="ttest", significance_level=0.95 ) raise "statistical-test: '"+test_method+"' does not exist" unless ReportStatisticalTest.respond_to?(test_method) grouped_validations = Reports::Util.group(validations, [group_attribute]) @@ -35,12 +35,20 @@ module Reports {:titles => titles, :matrix => matrix, :num_results => grouped_validations[0].size} end - def self.paired_ttest( validations1, validations2, attribute, class_value, significance_level=0.95 ) + def self.ttest( validations1, validations2, attribute, class_value, significance_level=0.95 ) array1 = validations1.collect{ |v| (v.send(attribute).is_a?(Hash) ? v.send(attribute)[class_value].to_f : v.send(attribute).to_f) } array2 = validations2.collect{ |v| (v.send(attribute).is_a?(Hash) ? v.send(attribute)[class_value].to_f : v.send(attribute).to_f) } LOGGER.debug "paired-t-testing "+attribute.to_s+" "+array1.inspect+" vs "+array2.inspect - Reports::r_util.paired_ttest(array1, array2, significance_level) + if array1.size>1 && array2.size>1 + Reports::r_util.paired_ttest(array1, array2, significance_level) + elsif array1.size==1 && array2.size>1 + -1 * Reports::r_util.ttest(array2, array1[0], significance_level) + elsif array1.size>1 && array2.size==1 + Reports::r_util.ttest(array1, array2[0], significance_level) + else + raise "illegal input for ttest" + end end end diff --git a/report/validation_access.rb b/report/validation_access.rb index e2a3978..4d9ed9f 100755 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -195,30 +195,50 @@ class Reports::ValidationDB Lib::OTPredictions.new( data.data, data.compounds ) end + @@accept_values = {} + def get_accept_values( validation, subjectid=nil ) - # PENDING So far, one has to load the whole dataset to get the accept_value from ambit - test_target_datasets = validation.test_target_dataset_uri - test_target_datasets = validation.test_dataset_uri unless test_target_datasets - res = nil - test_target_datasets.split(";").each do |test_target_dataset| - d = Lib::DatasetCache.find( test_target_dataset, subjectid ) - raise "cannot get test target dataset for accept values, dataset: "+test_target_dataset.to_s unless d - accept_values = d.accept_values(validation.prediction_feature) - raise "cannot get accept values from dataset "+test_target_dataset.to_s+" for feature "+ - validation.prediction_feature+":\n"+d.features[validation.prediction_feature].to_yaml unless accept_values!=nil - raise "different accept values" if res && res!=accept_values - res = accept_values + begin + return @@accept_values[validation.prediction_feature] if @@accept_values[validation.prediction_feature] + LOGGER.debug "get accept values ..." + pred = OpenTox::Feature.find(validation.prediction_feature) + accept = pred.metadata[OT.acceptValue] + accept = accept[0] if accept.is_a?(Array) and accept.size==1 and accept[0].is_a?(Array) + raise unless accept.is_a?(Array) and accept.size>1 + @@accept_values[validation.prediction_feature] = accept + LOGGER.debug "get accept values ... #{accept} #{accept.size}" + accept + rescue + # PENDING So far, one has to load the whole dataset to get the accept_value from ambit + test_target_datasets = validation.test_target_dataset_uri + test_target_datasets = validation.test_dataset_uri unless test_target_datasets + res = nil + test_target_datasets.split(";").each do |test_target_dataset| + d = Lib::DatasetCache.find( test_target_dataset, subjectid ) + raise "cannot get test target dataset for accept values, dataset: "+test_target_dataset.to_s unless d + accept_values = d.accept_values(validation.prediction_feature) + raise "cannot get accept values from dataset "+test_target_dataset.to_s+" for feature "+ + validation.prediction_feature+":\n"+d.features[validation.prediction_feature].to_yaml unless accept_values!=nil + raise "different accept values" if res && res!=accept_values + res = accept_values + end + res end - res end def feature_type( validation, subjectid=nil ) - OpenTox::Model::Generic.new(validation.model_uri).feature_type(subjectid) + if validation.model_uri.include?(";") + model_uri = validation.model_uri.split(";")[0] + else + model_uri = validation.model_uri + end + OpenTox::Model::Generic.new(model_uri).feature_type(subjectid) #get_model(validation).classification? end def predicted_variable(validation, subjectid=nil) raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) + raise "multiple models in this validation, cannot get one predicted variable (#{validation.model_uri})" if validation.model_uri.include?(";") model = OpenTox::Model::Generic.find(validation.model_uri, subjectid) raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model model.predicted_variable(subjectid) @@ -226,6 +246,7 @@ class Reports::ValidationDB def predicted_confidence(validation, subjectid=nil) raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) + raise "multiple models in this validation, cannot get one predicted confidence (#{validation.model_uri})" if validation.model_uri.include?(";") model = OpenTox::Model::Generic.find(validation.model_uri, subjectid) raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model model.predicted_confidence(subjectid) diff --git a/report/validation_data.rb b/report/validation_data.rb index 3806fd7..95636fc 100755 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -396,6 +396,7 @@ module Reports @validations.each do |v| index = -1 array.push(attributes.collect do |a| + index += 1 if VAL_ATTR_VARIANCE.index(a) variance = v.send( (a.to_s+"_variance").to_sym ) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 1bc55f6..89f47e0 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -9,8 +9,8 @@ require 'validation/validation_service.rb' helpers do def check_stratified(params) params[:stratified] = "false" unless params[:stratified] - raise OpenTox::BadRequestError.new "stratified != true|false|super, is #{params[:stratified]}" unless - params[:stratified]=~/true|false|super/ + raise OpenTox::BadRequestError.new "stratified != true|false|super|anti, is #{params[:stratified]}" unless + params[:stratified]=~/true|false|super|anti/ end end @@ -22,7 +22,7 @@ get '/crossvalidation/?' do params[:algorithm] = model.metadata[OT.algorithm] params[:dataset] = model.metadata[OT.trainingDataset] end - uri_list = Lib::OhmUtil.find( Validation::Crossvalidation, params ).sort.collect{|v| v.crossvalidation_uri}.join("\n") + "\n" + uri_list = Lib::OhmUtil.find( Validation::Crossvalidation, params ).sort.delete_if{|v| !v.finished}.collect{|v| v.crossvalidation_uri}.join("\n") + "\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "Single validations: "+url_for("/",:full)+"\n"+ @@ -204,6 +204,18 @@ get '/crossvalidation/:id/statistics' do end end +get '/crossvalidation/:id/statistics/uri' do + uri = Validation::Validation.find( :crossvalidation_id => params[:id], :validation_type => "crossvalidation_statistics" ).first.validation_uri + case request.env['HTTP_ACCEPT'].to_s + when /text\/html/ + content_type "text/html" + OpenTox.text_to_html uri + else + content_type "text/uri-list" + uri + end +end + get '/crossvalidation/:id/statistics/probabilities' do LOGGER.info "get crossvalidation statistics for crossvalidation with id "+params[:id].to_s @@ -273,7 +285,7 @@ end get '/?' do LOGGER.info "list all validations, params: "+params.inspect - uri_list = Lib::OhmUtil.find( Validation::Validation, params ).sort.collect{|v| v.validation_uri}.join("\n") + "\n" + uri_list = Lib::OhmUtil.find( Validation::Validation, params ).sort.delete_if{|v| !v.finished}.collect{|v| v.validation_uri}.join("\n") + "\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "To perform a validation:\n"+ @@ -619,6 +631,58 @@ get '/:id/probabilities' do end +get '/:id/viz' do + + begin + validation = Validation::Validation.get(params[:id]) + rescue ActiveRecord::RecordNotFound => ex + raise OpenTox::NotFoundError.new("Validation '#{params[:id]}' not found.") + end + + m = OpenTox::Model::Generic.find(validation.model_uri) + predicted_feature = m.predicted_variable(nil) + actual_feature = validation.prediction_feature + + d = OpenTox::Dataset.create + + test = OpenTox::Dataset.find(validation.test_dataset_uri) + training = OpenTox::Dataset.find(validation.training_dataset_uri) + prediction = OpenTox::Dataset.find(validation.prediction_dataset_uri) + + dataset_feature = "http://dataset" + d.add_feature(dataset_feature) + correct_classified_feature = "http://correct-classified" + d.add_feature(correct_classified_feature) + predicted_nice_feature = "http://predicted" + d.add_feature(predicted_nice_feature) + + [training, test].each do |data| + data.compounds.each do |c| + d.add_compound(c) + d.add(c,dataset_feature,data==training ? "training" : "test") + data.features.each do |f,m| + d.add_feature(f,m) + data.data_entries[c][f].each do |v| + d.add(c,f,v) + end if data.data_entries[c][f] + end + end + end + + prediction.compounds.each do |c| + if prediction.data_entries[c][predicted_feature] + p = prediction.data_entries[c][predicted_feature] + a = d.data_entries[c][actual_feature] + [p,a].each do |v| + raise p.class.to_s+" "+p.inspect unless p.is_a?(Array) and p.size==1 + end + d.add(c,predicted_nice_feature,p[0]) + d.add(c,correct_classified_feature,p[0]==a[0] ? "correct" : "miss") + end + end + d.to_csv +end + #get '/:id/predictions' do # LOGGER.info "get validation predictions "+params.inspect # begin diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 8c8b11f..d30c7d2 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -446,6 +446,8 @@ module Validation meta = { DC.creator => self.crossvalidation_uri } case stratified + when "anti" + raise "anti-stratification not yet supported for cv" when "false" if self.loo=="true" shuffled_compounds = orig_dataset.compounds @@ -631,16 +633,16 @@ module Validation meta = { DC.creator => $url_provider.url_for('/training_test_split',:full) } case stratified - when /true|super/ + when /true|super|anti/ if stratified=="true" raise OpenTox::BadRequestError.new "prediction feature required for stratified splits" unless prediction_feature features = [prediction_feature] else - LOGGER.warn "prediction feature is ignored for super-stratified splits" if prediction_feature + LOGGER.warn "prediction feature is ignored for super- or anti-stratified splits" if prediction_feature features = nil end r_util = OpenTox::RUtil.new - train, test = r_util.stratified_split( orig_dataset, meta, "NA", split_ratio, @subjectid, random_seed, features ) + train, test = r_util.stratified_split( orig_dataset, meta, "NA", split_ratio, @subjectid, random_seed, features, stratified=="anti" ) r_util.quit_r result = {:training_dataset_uri => train.uri, :test_dataset_uri => test.uri} when "false" diff --git a/validation/validation_test.rb b/validation/validation_test.rb index 70f3ca4..0001076 100755 --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -84,9 +84,14 @@ class ValidationTest < Test::Unit::TestCase # post "/report/method_comparison", # {:validation_uris=>"http://local-ot/validation/389,http://local-ot/validation/390,http://local-ot/validation/391,http://local-ot/validation/392", # :identifier=>"split1,split1,split2,split2"} - - - #post "/report/validation",{:validation_uris=>"http://local-ot/validation/171"} + + get '15709/viz' + puts last_response.body + exit + + post "/report/validation",{:validation_uris=>"http://local-ot/validation/15647"} + exit + #post "/report/validation",{:validation_uris=>"http://local-ot/validation/389"} #dataset_uri = OpenTox::Dataset.create_from_csv_file(File.new("data/EPAFHM.csv").path, nil).uri @@ -94,9 +99,13 @@ class ValidationTest < Test::Unit::TestCase # #dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/603306?feature_uris[]=http://apps.ideaconsult.net:8080/ambit2/feature/764036" # #dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/603204" -# post "/plain_training_test_split",{:dataset_uri=>dataset_uri, :stratified=>"true", :split_ratio=>0.3} -# puts last_response.body -# uri = last_response.body + + dataset_uri = "http://local-ot/dataset/9264" + post "/plain_training_test_split",{:dataset_uri=>dataset_uri, :stratified=>"false", :split_ratio=>0.5} + puts last_response.body + uri = last_response.body + exit + # rep = wait_for_task(uri) # puts rep #OpenTox::RestClientWrapper.post("http://opentox.informatik.uni-freiburg.de/validation/plain_training_test_split", @@ -131,7 +140,8 @@ class ValidationTest < Test::Unit::TestCase #get 'crossvalidation/189/statistics' #puts last_response.body - #run_test("13a") + run_test("1a") + #run_test("13a",:validation_uri=>"http://local-ot/validation/1568") # run_test("1a",:validation_uri=>"http://local-ot/validation/513") #get '/crossvalidation/79/predictions',nil,'HTTP_ACCEPT' => "application/x-yaml" @@ -178,8 +188,8 @@ class ValidationTest < Test::Unit::TestCase # :random_seed => 1 # }) - #run_test("23a") - run_test("23a",{:validation_uri=>"http://local-ot/validation/crossvalidation/53"}) + #run_test("3a") + #run_test("23a",{:validation_uri=>"http://local-ot/validation/crossvalidation/56"}) #run_test("23a",{:validation_uri=>"http://local-ot/validation/crossvalidation/47"}) #23a loo {:validation_uri=>"http://local-ot/validation/crossvalidation/47"}) #loo mit datasets auf ortona {:validation_uri=>"http://local-ot/validation/crossvalidation/46"} -- cgit v1.2.3