diff options
author | mguetlein <martin.guetlein@gmail.com> | 2011-05-18 15:35:52 +0200 |
---|---|---|
committer | mguetlein <martin.guetlein@gmail.com> | 2011-05-18 15:35:52 +0200 |
commit | ac6a536d12697a98db6847c5229c67372cbbd1e7 (patch) | |
tree | 1b5bfaabceb8a8d809724e91d5474ac7f66ef7be | |
parent | 9ce03c0f50bb9129b584327d56fa4c9277849227 (diff) |
new feature: algorithm comparison report
-rw-r--r-- | lib/merge.rb | 2 | ||||
-rwxr-xr-x | lib/predictions.rb | 19 | ||||
-rw-r--r-- | report/plot_factory.rb | 34 | ||||
-rwxr-xr-x | report/report_application.rb | 3 | ||||
-rwxr-xr-x | report/report_content.rb | 61 | ||||
-rwxr-xr-x | report/report_factory.rb | 102 | ||||
-rwxr-xr-x | report/report_persistance.rb | 1 | ||||
-rw-r--r-- | report/report_service.rb | 7 | ||||
-rw-r--r-- | report/statistical_test.rb | 2 | ||||
-rwxr-xr-x | report/validation_access.rb | 12 | ||||
-rwxr-xr-x | report/validation_data.rb | 126 | ||||
-rwxr-xr-x | report/xml_report.rb | 16 | ||||
-rwxr-xr-x | test/test_examples.rb | 210 | ||||
-rwxr-xr-x | validation/validation_service.rb | 6 | ||||
-rwxr-xr-x | validation/validation_test.rb | 283 |
15 files changed, 599 insertions, 285 deletions
diff --git a/lib/merge.rb b/lib/merge.rb index ecbe133..f30a3c1 100644 --- a/lib/merge.rb +++ b/lib/merge.rb @@ -126,7 +126,7 @@ module Lib if value1==nil && value2==nil value = nil elsif value1.to_s != value2.to_s - value = value1.to_s + "/" + value2.to_s + value = value1.to_s + ";" + value2.to_s else value = value2.to_s end diff --git a/lib/predictions.rb b/lib/predictions.rb index 420790e..f97b764 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -170,8 +170,8 @@ module Lib def percent_correct raise "no classification" unless @feature_type=="classification" - return 0 if @num_with_actual_value==0 - return 100 * @num_correct / (@num_with_actual_value - @num_unpredicted).to_f + pct = 100 * @num_correct / (@num_with_actual_value - @num_unpredicted).to_f + pct.nan? ? 0 : pct end def percent_incorrect @@ -181,7 +181,8 @@ module Lib end def accuracy - return percent_correct / 100.0 + acc = percent_correct / 100.0 + acc.nan? ? 0 : acc end def weighted_accuracy @@ -250,6 +251,7 @@ module Lib return res end + # does only take the instances that are classified as <class-index> into account def area_under_roc(class_index=nil) return prediction_feature_value_map( lambda{ |i| area_under_roc(i) } ) if class_index==nil @@ -427,8 +429,13 @@ module Lib return incorrect end + # Note: + # * (un-weighted) area under roc is computed with all __predicted__ isntances for a certain class + # * weighted weights each auc with the number of __acutal__ instances + # its like that, because its like that in weka def weighted_area_under_roc - return weighted_measure( :area_under_roc ) + w_auc = weighted_measure( :area_under_roc ) + w_auc.nan? ? 0 : w_auc end def weighted_f_measure @@ -436,6 +443,7 @@ module Lib end private + # the <measure> is weighted with the number of instances for each actual class value def weighted_measure( measure ) sum_instances = 0 @@ -478,9 +486,10 @@ module Lib def sample_correlation_coefficient # formula see http://en.wikipedia.org/wiki/Correlation_and_dependence#Pearson.27s_product-moment_coefficient - return ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) / + scc = ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) / ( Math.sqrt( [0, @num_predicted * @sum_squares_actual - @sum_actual**2].max ) * Math.sqrt( [0, @num_predicted * @sum_squares_predicted - @sum_predicted**2].max ) ) + ( scc.infinite? || scc.nan? ) ? 0 : scc end def total_sum_of_squares diff --git a/report/plot_factory.rb b/report/plot_factory.rb index b7c920a..74c89f5 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -167,25 +167,35 @@ module Reports validation_set.validations.each do |v| values = [] value_attributes.each do |a| - validation_set.get_accept_values_for_attr(a).each do |class_value| - value = v.send(a) - if value.is_a?(Hash) - if class_value==nil - avg_value = 0 - value.values.each{ |val| avg_value+=val } - value = avg_value/value.values.size.to_f - else - raise "bar plot value is hash, but no entry for class-value ("+class_value.to_s+"); value for "+a.to_s+" -> "+value.inspect unless value.key?(class_value) - value = value[class_value] + + accept = validation_set.get_accept_values_for_attr(a) + if accept and accept.size>0 + accept.each do |class_value| + value = v.send(a) + if value.is_a?(Hash) + if class_value==nil + avg_value = 0 + value.values.each{ |val| avg_value+=val } + value = avg_value/value.values.size.to_f + else + raise "bar plot value is hash, but no entry for class-value ("+class_value.to_s+"); value for "+a.to_s+" -> "+value.inspect unless value.key?(class_value) + value = value[class_value] + end end + raise "value is nil\nattribute: "+a.to_s+"\nvalidation: "+v.inspect if value==nil + values.push(value) + labels.push(a.to_s.gsub("_","-") + ( class_value==nil ? "" : "("+class_value.to_s+")" )) end - raise "value is nil\nattribute: "+a.to_s+"\nvalidation: "+v.inspect if value==nil + else + value = v.send(a) values.push(value) - labels.push(a.to_s.gsub("_","-") + ( class_value==nil ? "" : "("+class_value.to_s+")" )) + labels.push(a.to_s.gsub("_","-")) end + end titles << v.send(title_attribute).to_s + raise "no title for '"+title_attribute.to_s+"' in validation: "+v.to_yaml if titles[-1].to_s.size==0 data << values end diff --git a/report/report_application.rb b/report/report_application.rb index 258daa7..1637ead 100755 --- a/report/report_application.rb +++ b/report/report_application.rb @@ -114,7 +114,8 @@ end post '/report/:type' do task = OpenTox::Task.create("Create report",url_for("/report/"+params[:type], :full)) do |task| #,params perform do |rs| - rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil,@subjectid,task) + rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil, + params[:identifier]?params[:identifier].split(/\n|,/):nil,@subjectid,task) end end return_task(task) diff --git a/report/report_content.rb b/report/report_content.rb index cc4c13c..ac64bab 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -22,28 +22,32 @@ class Reports::ReportContent @current_section = @xml_report.get_root_element end - def add_paired_ttest_table( validation_set, + def add_paired_ttest_tables( validation_set, group_attribute, - test_attribute, + test_attributes, section_title = "Paired t-test", section_text = nil) - - level = 0.90 - test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, - group_attribute, test_attribute, "paired_ttest", level ) - #puts test_matrix.inspect - titles = test_matrix[:titles] - matrix = test_matrix[:matrix] - table = [] - #puts titles.inspect - table << [""] + titles - titles.size.times do |i| - table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") } - end - + section_test = @xml_report.add_section(@current_section, section_title) @xml_report.add_paragraph(section_test, section_text) if section_text - @xml_report.add_table(section_test, test_attribute.to_s+", significance-level: "+level.to_s, table, true, true) + + test_attributes.each do |test_attribute| + level = 0.90 + test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, + group_attribute, test_attribute, "paired_ttest", level ) + #puts test_matrix.inspect + titles = test_matrix[:titles] + matrix = test_matrix[:matrix] + table = [] + #puts titles.inspect + table << [""] + titles + titles.size.times do |i| + table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") } + end + + @xml_report.add_table(section_test, test_attribute.to_s+", significance-level: "+level.to_s+", num results: "+ + test_matrix[:num_results].to_s, table, true, true) + end end def add_predictions( validation_set, @@ -98,32 +102,13 @@ class Reports::ReportContent validation_attributes, table_title, section_title="Results", - section_text=nil, - #rem_equal_vals_attr=[], - search_for_existing_report_type=nil) + section_text=nil) + #rem_equal_vals_attr=[]) section_table = @xml_report.add_section(@current_section, section_title) @xml_report.add_paragraph(section_table, section_text) if section_text vals = validation_set.to_array(validation_attributes, true) vals = vals.collect{|a| a.collect{|v| v.to_s }} - - if (search_for_existing_report_type) - vals.size.times do |i| - #puts i - if (i==0) - vals[i] = [ "Reports" ] + vals[i] - puts vals[i].inspect - else - if search_for_existing_report_type=="validation" - vals[i] = [ validation_set.validations[i-1].validation_report_uri() ] + vals[i] - elsif search_for_existing_report_type=="crossvalidation" - vals[i] = [ validation_set.validations[i-1].cv_report_uri() ] + vals[i] - else - raise "illegal report type: "+search_for_existing_report_type.to_s - end - end - end - end #PENDING transpose values if there more than 4 columns, and there are more than columns than rows transpose = vals[0].size>4 && vals[0].size>vals.size @xml_report.add_table(section_table, table_title, vals, !transpose, transpose, transpose) diff --git a/report/report_factory.rb b/report/report_factory.rb index 08d9418..7e74cb4 100755 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -100,33 +100,37 @@ module Reports::ReportFactory raise OpenTox::BadRequestError.new("validations must have unique feature type, i.e. must be either all regression, "+ +"or all classification validations") unless validation_set.unique_feature_type pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) ) + validation_set.validations.sort! do |x,y| + x.crossvalidation_fold.to_f <=> y.crossvalidation_fold.to_f + end + cv_set = validation_set.replace_with_cv_stats + raise unless cv_set.size==1 - merged = validation_set.merge([:crossvalidation_id]) - raise unless merged.size==1 - - #puts merged.get_values(:percent_correct_variance, false).inspect + #puts cv_set.get_values(:percent_correct_variance, false).inspect report = Reports::ReportContent.new("Crossvalidation report") + res_titel = "Crossvalidation Results" + res_text = "These performance statistics have been derieved by accumulating all predictions on the various fold (i.e. these numbers are NOT averaged results over all crossvalidation folds)." case validation_set.unique_feature_type when "classification" - report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold],"Mean Results","Mean Results") - report.add_confusion_matrix(merged.validations[0]) + report.add_result(cv_set, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold], res_titel, res_titel, res_text) + report.add_confusion_matrix(cv_set.validations[0]) report.add_section("Plots") report.add_roc_plot(validation_set) report.add_roc_plot(validation_set, :crossvalidation_fold) report.add_confidence_plot(validation_set) report.add_confidence_plot(validation_set, :crossvalidation_fold) report.end_section - report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_CLASS-[:num_folds], - "Results","Results",nil,"validation") + report.add_result(validation_set, [:validation_uri, :validation_report_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:num_folds, :dataset_uri, :algorithm_uri], + "Results","Results") when "regression" - report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold],"Mean Results","Mean Results") + report.add_result(cv_set, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold],res_titel, res_titel, res_text) report.add_section("Plots") report.add_regression_plot(validation_set, :crossvalidation_fold) report.add_confidence_plot(validation_set) report.add_confidence_plot(validation_set, :crossvalidation_fold) report.end_section - report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_REGR-[:num_folds], "Results","Results") + report.add_result(validation_set, [:validation_uri, :validation_report_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:num_folds, :dataset_uri, :algorithm_uri], "Results","Results") end task.progress(90) if task @@ -142,8 +146,8 @@ module Reports::ReportFactory raise OpenTox::BadRequestError.new("num validations is not >1") unless validation_set.size>1 raise OpenTox::BadRequestError.new("validations must have unique feature type, i.e. must be either all regression, "+ +"or all classification validations") unless validation_set.unique_feature_type - raise OpenTox::BadRequestError.new("number of different algorithms <2: "+ - validation_set.get_values(:algorithm_uri).inspect) if validation_set.num_different_values(:algorithm_uri)<2 + raise OpenTox::BadRequestError.new("number of different identifiers <2: "+ + validation_set.get_values(:identifier).inspect) if validation_set.num_different_values(:identifier)<2 if validation_set.has_nil_values?(:crossvalidation_id) raise OpenTox::BadRequestError.new("algorithm comparison for non crossvalidation not yet implemented") @@ -160,73 +164,63 @@ module Reports::ReportFactory # groups results into sets with equal dataset if (validation_set.num_different_values(:dataset_uri)>1) + LOGGER.debug "compare report -- num different datasets: "+validation_set.num_different_values(:dataset_uri).to_s dataset_grouping = Reports::Util.group(validation_set.validations, [:dataset_uri]) # check if equal values in each group exist - Reports::Util.check_group_matching(dataset_grouping, [:algorithm_uri, :crossvalidation_fold, :num_folds, :stratified, :random_seed]) + Reports::Util.check_group_matching(dataset_grouping, [:crossvalidation_fold, :num_folds, :stratified, :random_seed]) else dataset_grouping = [ validation_set.validations ] end - # we only checked that equal validations exist in each dataset group, now check for each algorithm + # we only checked that equal validations exist in each dataset group, now check for each identifier dataset_grouping.each do |validations| - algorithm_grouping = Reports::Util.group(validations, [:algorithm_uri]) + algorithm_grouping = Reports::Util.group(validations, [:identifier]) Reports::Util.check_group_matching(algorithm_grouping, [:crossvalidation_fold, :num_folds, :stratified, :random_seed]) end pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) ) - report = Reports::ReportContent.new("Algorithm comparison report - Many datasets") + report = Reports::ReportContent.new("Algorithm comparison report") if (validation_set.num_different_values(:dataset_uri)>1) all_merged = validation_set.merge([:algorithm_uri, :dataset_uri, :crossvalidation_id, :crossvalidation_uri]) report.add_ranking_plots(all_merged, :algorithm_uri, :dataset_uri, [:percent_correct, :weighted_area_under_roc, :true_positive_rate, :true_negative_rate] ) report.add_result_overview(all_merged, :algorithm_uri, :dataset_uri, [:percent_correct, :weighted_area_under_roc, :true_positive_rate, :true_negative_rate]) - end - + + result_attributes = [:identifier,:crossvalidation_uri,:crossvalidation_report_uri]+VAL_ATTR_CV-[:crossvalidation_fold,:num_folds,:dataset_uri] case validation_set.unique_feature_type when "classification" - attributes = VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold] - attributes = ([ :dataset_uri ] + attributes).uniq - - dataset_grouping.each do |validations| - - set = Reports::ValidationSet.create(validations) - - dataset = validations[0].dataset_uri - merged = set.merge([:algorithm_uri, :dataset_uri, :crossvalidation_id, :crossvalidation_uri]) - merged.sort(:dataset_uri) - - report.add_section("Dataset: "+dataset) - report.add_result(merged,attributes, - "Mean Results","Mean Results",nil,"crossvalidation") - report.add_paired_ttest_table(set, :algorithm_uri, :percent_correct) - - report.add_bar_plot(merged, :algorithm_uri, VAL_ATTR_BAR_PLOT_CLASS) - report.add_roc_plot(set, :algorithm_uri) - report.end_section - end - - when "regression" + result_attributes += VAL_ATTR_CLASS + ttest_attributes = [:percent_correct, :weighted_area_under_roc] + bar_plot_attributes = VAL_ATTR_BAR_PLOT_CLASS + else + result_attributes += VAL_ATTR_REGR + ttest_attributes = [:r_square, :root_mean_squared_error] + bar_plot_attributes = VAL_ATTR_BAR_PLOT_REGR + end - attributes = VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold] - attributes = ([ :dataset_uri ] + attributes).uniq + dataset_grouping.each do |validations| + + set = Reports::ValidationSet.create(validations) - dataset_grouping.each do |validations| + dataset = validations[0].dataset_uri + merged = set.merge([:identifier, :dataset_uri]) #, :crossvalidation_id, :crossvalidation_uri]) + merged.sort(:identifier) - set = Reports::ValidationSet.create(validations) - - dataset = validations[0].dataset_uri - merged = set.merge([:algorithm_uri, :dataset_uri, :crossvalidation_id, :crossvalidation_uri]) - merged.sort(:dataset_uri) - - report.add_section("Dataset: "+dataset) - report.add_result(merged,attributes, - "Mean Results","Mean Results",nil,"crossvalidation") - report.add_paired_ttest_table(set, :algorithm_uri, :r_square) - report.end_section + merged.validations.each do |v| + v.crossvalidation_uri = v.crossvalidation_uri.split(";").uniq.join(" ") + v.crossvalidation_report_uri = v.crossvalidation_report_uri.split(";").uniq.join(" ") if v.crossvalidation_report_uri end + report.add_section("Dataset: "+dataset) + res_titel = "Average Results on Folds" + res_text = "These performance statistics have been derieved by computing the mean of the statistics on each crossvalidation fold." + report.add_result(merged,result_attributes,res_titel,res_titel,res_text) + # pending: regression stats have different scales!!! + report.add_bar_plot(merged, :identifier, bar_plot_attributes) if validation_set.unique_feature_type=="classification" + report.add_paired_ttest_tables(set, :identifier, ttest_attributes) + report.end_section end task.progress(100) if task report diff --git a/report/report_persistance.rb b/report/report_persistance.rb index c85ad68..78ae47b 100755 --- a/report/report_persistance.rb +++ b/report/report_persistance.rb @@ -250,6 +250,7 @@ module Reports end def list_reports(type, filter_params={}) + filter_params[:report_type] = type LOGGER.debug "find reports for params: "+filter_params.inspect reports = Lib::OhmUtil.find( ReportData, filter_params ) reports.collect{ |r| r.id } diff --git a/report/report_service.rb b/report/report_service.rb index 722c3d6..3e23889 100644 --- a/report/report_service.rb +++ b/report/report_service.rb @@ -60,7 +60,7 @@ module Reports # call-seq: # create_report(type, validation_uris) => string # - def create_report(type, validation_uris, subjectid=nil, task=nil) + def create_report(type, validation_uris, identifier=nil, subjectid=nil, task=nil) LOGGER.info "create report of type '"+type.to_s+"'" check_report_type(type) @@ -68,7 +68,10 @@ module Reports # step1: load validations raise OpenTox::BadRequestError.new("validation_uris missing") unless validation_uris LOGGER.debug "validation_uri(s): '"+validation_uris.inspect+"'" - validation_set = Reports::ValidationSet.new(validation_uris, subjectid) + LOGGER.debug "identifier: '"+identifier.inspect+"'" + raise "illegal num identifiers: "+identifier.size.to_s+" should be equal to num validation-uris ("+validation_uris.size.to_s+")" if + identifier and identifier.size!=validation_uris.size + validation_set = Reports::ValidationSet.new(validation_uris, identifier, subjectid) raise OpenTox::BadRequestError.new("cannot get validations from validation_uris '"+validation_uris.inspect+"'") unless validation_set and validation_set.size > 0 LOGGER.debug "loaded "+validation_set.size.to_s+" validation/s" task.progress(10) if task diff --git a/report/statistical_test.rb b/report/statistical_test.rb index c37827e..1e586e2 100644 --- a/report/statistical_test.rb +++ b/report/statistical_test.rb @@ -58,7 +58,7 @@ module Reports end end end - {:titles => titles, :matrix => matrix} + {:titles => titles, :matrix => matrix, :num_results => grouped_validations[0].size} end def self.paired_ttest( validations1, validations2, attribute, significance_level=0.95 ) diff --git a/report/validation_access.rb b/report/validation_access.rb index d0c3a1d..ffb7461 100755 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -7,8 +7,9 @@ require "lib/validation_db.rb" # class Reports::ValidationDB - def resolve_cv_uris(validation_uris, subjectid=nil) - res = [] + def resolve_cv_uris(validation_uris, identifier=nil, subjectid=nil) + res = {} + count = 0 validation_uris.each do |u| if u.to_s =~ /.*\/crossvalidation\/[0-9]+/ cv_id = u.split("/")[-1].to_i @@ -25,10 +26,13 @@ class Reports::ValidationDB raise OpenTox::NotFoundError.new "crossvalidation with id "+cv_id.to_s+" not found" unless cv raise OpenTox::BadRequestError.new("crossvalidation with id '"+cv_id.to_s+"' not finished") unless cv.finished #res += Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s} - res += Validation::Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|v| v.validation_uri.to_s } + Validation::Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).each do |v| + res[v.validation_uri.to_s] = identifier ? identifier[count] : nil + end else - res += [u.to_s] + res[u.to_s] = identifier ? identifier[count] : nil end + count += 1 end res end diff --git a/report/validation_data.rb b/report/validation_data.rb index 11fa737..0c15dd2 100755 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -1,6 +1,7 @@ # the variance is computed when merging results for these attributes -VAL_ATTR_VARIANCE = [ :area_under_roc, :percent_correct, :root_mean_squared_error, :mean_absolute_error, :r_square, :accuracy ] +VAL_ATTR_VARIANCE = [ :area_under_roc, :percent_correct, :root_mean_squared_error, :mean_absolute_error, + :r_square, :accuracy, :weighted_area_under_roc, :weighted_accuracy ] VAL_ATTR_RANKING = [ :area_under_roc, :percent_correct, :true_positive_rate, :true_negative_rate, :weighted_area_under_roc ] #:accuracy ] ATTR_NICE_NAME = {} @@ -51,21 +52,31 @@ end module Reports + @@validation_access = ValidationDB.new + @@persistance = ReportService.persistance + + def self.persistance + @@persistance + end + + def self.validation_access + @@validation_access + end + + # for overwriting validation source (other than using webservices) + def self.reset_validation_access(validation_access) + @@validation_access = validation_access + end + + # = ReportValidation # # contains all values of a validation object # class ReportValidation - @@validation_access = ValidationDB.new - - # for overwriting validation source (other than using webservices) - def self.reset_validation_access(validation_access) - @@validation_access = validation_access - end - - def self.resolve_cv_uris(validation_uris, subjectid) - @@validation_access.resolve_cv_uris(validation_uris, subjectid) + def self.resolve_cv_uris(validation_uris, identifier, subjectid) + Reports.validation_access.resolve_cv_uris(validation_uris, identifier, subjectid) end # create member variables for all validation properties @@ -75,16 +86,17 @@ module Reports @@validation_attributes.each{ |a| attr_accessor a } attr_reader :predictions + attr_accessor :identifier, :validation_report_uri, :crossvalidation_report_uri def initialize(uri = nil, subjectid = nil) - @@validation_access.init_validation(self, uri, subjectid) if uri + Reports.validation_access.init_validation(self, uri, subjectid) if uri @subjectid = subjectid #raise "subjectid is nil" unless subjectid end def self.from_cv_statistics( cv_uri, subjectid = nil ) v = ReportValidation.new(nil, subjectid) - @@validation_access.init_validation_from_cv_statistics(v, cv_uri, subjectid) + Reports.validation_access.init_validation_from_cv_statistics(v, cv_uri, subjectid) v end @@ -103,7 +115,7 @@ module Reports task.progress(100) if task nil else - @predictions = @@validation_access.get_predictions( self, @subjectid, task ) + @predictions = Reports.validation_access.get_predictions( self, @subjectid, task ) end end end @@ -111,7 +123,7 @@ module Reports # returns the predictions feature values (i.e. the domain of the class attribute) # def get_accept_values() - @accept_values = @@validation_access.get_accept_values(self, @subjectid) unless @accept_values + @accept_values = Reports.validation_access.get_accept_values(self, @subjectid) unless @accept_values @accept_values end @@ -119,36 +131,21 @@ module Reports # def feature_type return @feature_type if @feature_type!=nil - @feature_type = @@validation_access.feature_type(self, @subjectid) + @feature_type = Reports.validation_access.feature_type(self, @subjectid) end def predicted_variable return @predicted_variable if @predicted_variable!=nil - @predicted_variable = @@validation_access.predicted_variable(self, @subjectid) + @predicted_variable = Reports.validation_access.predicted_variable(self, @subjectid) end # loads all crossvalidation attributes, of the corresponding cv into this object def load_cv_attributes raise "crossvalidation-id not set" unless @crossvalidation_id - @@validation_access.init_cv(self) - end - - @@persistance = ReportService.persistance - - def validation_report_uri - #puts "searching for validation report: "+self.validation_uri.to_s - return @validation_report_uri if @validation_report_uri!=nil - ids = @@persistance.list_reports("validation",{:validation_uris=>validation_uri }) - @validation_report_uri = ReportService.instance.get_uri("validation",ids[-1]) if ids and ids.size>0 - end - - def cv_report_uri - #puts "searching for cv report: "+self.crossvalidation_uri.to_s - return @cv_report_uri if @cv_report_uri!=nil - raise "no cv uri "+to_yaml unless self.crossvalidation_uri - ids = @@persistance.list_reports("crossvalidation",{:crossvalidation=>self.crossvalidation_uri.to_s }) - #puts "-> "+ids.inspect - @cv_report_uri = ReportService.instance.get_uri("crossvalidation",ids[-1]) if ids and ids.size>0 + Reports.validation_access.init_cv(self) + # load cv report + ids = Reports.persistance.list_reports("crossvalidation",{:crossvalidation=>self.crossvalidation_uri.to_s }) + @crossvalidation_report_uri = ReportService.instance.get_uri("crossvalidation",ids[-1]) if ids and ids.size>0 end def clone_validation @@ -164,13 +161,20 @@ module Reports # class ValidationSet - def initialize(validation_uris=nil, subjectid=nil) + def initialize(validation_uris=nil, identifier=nil, subjectid=nil) @unique_values = {} - validation_uris = ReportValidation.resolve_cv_uris(validation_uris, subjectid) if validation_uris - @validations = Array.new - validation_uris.each{|u| @validations.push(ReportValidation.new(u, subjectid))} if validation_uris + @validations = [] + if validation_uris + validation_uri_and_ids = ReportValidation.resolve_cv_uris(validation_uris, identifier, subjectid) + validation_uri_and_ids.each do |u,id| + v = ReportValidation.new(u, subjectid) + v.identifier = id if id + ids = Reports.persistance.list_reports("validation",{:validation_uris=>v.validation_uri }) + v.validation_report_uri = ReportService.instance.get_uri("validation",ids[-1]) if ids and ids.size>0 + @validations << v + end + end end - def self.create(validations) set = ValidationSet.new @@ -399,6 +403,17 @@ module Reports return array end + def replace_with_cv_stats + new_set = ValidationSet.new + grouping = Util.group(@validations, [:crossvalidation_id]) + grouping.each do |g| + v = ReportValidation.from_cv_statistics(g[0].crossvalidation_uri) + v.identifier = g.collect{|vv| vv.identifier}.uniq.join(";") + new_set.validations << v + end + return new_set + end + # creates a new validaiton set, that contains merged validations # all validation with equal values for __equal_attributes__ are summed up in one validation, i.e. merged # @@ -416,31 +431,16 @@ module Reports grouping = Util.group(@validations, equal_attributes) #puts "groups "+grouping.size.to_s - if ( equal_attributes.include?(:crossvalidation_id) ) - # do not merge, use crossvalidation statistics - raise "statistics vs merging problem" if equal_attributes.size!=1 - grouping.each do |g| - new_set.validations << ReportValidation.from_cv_statistics(g[0].crossvalidation_uri) - end - else - #merge - Lib::MergeObjects.register_merge_attributes( ReportValidation, - Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL) unless - Lib::MergeObjects.merge_attributes_registered?(ReportValidation) - grouping.each do |g| - new_set.validations << g[0].clone_validation - w = 1 - g[1..-1].each do |v| - new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v,w,1) - w+=1 - end + #merge + Lib::MergeObjects.register_merge_attributes( ReportValidation, + Validation::VAL_MERGE_AVG+Validation::VAL_MERGE_SUM,[],Validation::VAL_MERGE_GENERAL+[:identifier, :validation_report_uri, :crossvalidation_report_uri]) unless + Lib::MergeObjects.merge_attributes_registered?(ReportValidation) + grouping.each do |g| + new_set.validations << g[0].clone_validation + g[1..-1].each do |v| + new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v) end end - - new_set.validations.each do |v| - raise "not a validation "+v.class.to_s+" "+v.to_s unless v.is_a?(Reports::ReportValidation) - end - return new_set end diff --git a/report/xml_report.rb b/report/xml_report.rb index 4fbfae3..7467c47 100755 --- a/report/xml_report.rb +++ b/report/xml_report.rb @@ -187,7 +187,13 @@ module Reports if auto_link_urls && v.to_s =~ /depict/ || v.to_s =~ /image\/png$/ #PENDING add_image(entry, v.to_s) elsif auto_link_urls && v.to_s =~ /^http(s?):\/\// - add_url(entry, v.to_s, v.to_s) + #add_url(entry, v.to_s, v.to_s) + v.to_s.split(" ").each do |vv| + add_url(entry, vv.to_s, vv.to_s) + space = Element.new("para") + space.text = " " + entry << space + end else entry.text = v.to_s end @@ -221,11 +227,15 @@ module Reports return list end - def add_url (element, url, description=url ) - + def url_element( url, description=url ) ulink = Element.new("ulink") ulink.add_attributes({"url" => url}) ulink.text = description + ulink + end + + def add_url (element, url, description=url ) + ulink = url_element(url, description) element << ulink return ulink end diff --git a/test/test_examples.rb b/test/test_examples.rb index 49d7838..eb0543f 100755 --- a/test/test_examples.rb +++ b/test/test_examples.rb @@ -74,6 +74,14 @@ module ValidationExamples end end + class LazarLastEPAFHMSplit < LazarEPAFHMSplit + def initialize + super + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/last") + end + end + + class MajorityEPAFHMSplit < EPAFHMSplit def initialize @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") @@ -81,11 +89,18 @@ module ValidationExamples end end + class MajorityRandomEPAFHMSplit < MajorityEPAFHMSplit + def initialize + @algorithm_params = "random=true" + super + end + end + ######################################################################################################## class EPAFHMCrossvalidation < CrossValidation def initialize - @dataset_file = File.new("data/EPAFHM.mini.csv","r") + @dataset_file = File.new("data/EPAFHM.med.csv","r") #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" @num_folds = 10 end @@ -93,11 +108,20 @@ module ValidationExamples class MajorityEPAFHMCrossvalidation < EPAFHMCrossvalidation def initialize + #@dataset_uri = "http://local-ot/dataset/2366" + #@prediction_feature = "http://local-ot/dataset/2366/feature/LC50_mmol" @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") super end end + class MajorityRandomEPAFHMCrossvalidation < MajorityEPAFHMCrossvalidation + def initialize + @algorithm_params = "random=true" + super + end + end + class LazarEPAFHMCrossvalidation < EPAFHMCrossvalidation def initialize @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") @@ -125,6 +149,14 @@ module ValidationExamples super end end + + class LazarLastHamsterSplit < LazarHamsterSplit + def initialize + super + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/last") + end + end + class MajorityHamsterSplit < HamsterSplit def initialize @@ -133,6 +165,13 @@ module ValidationExamples end end + class MajorityRandomHamsterSplit < MajorityHamsterSplit + def initialize + @algorithm_params = "random=true" + super + end + end + ######################################################################################################## class HamsterBootstrapping < BootstrappingValidation @@ -212,6 +251,13 @@ module ValidationExamples super end end + + class MajorityRandomHamsterCrossvalidation < MajorityHamsterCrossvalidation + def initialize + @algorithm_params = "random=true" + super + end + end class LazarHamsterCrossvalidation < HamsterCrossvalidation def initialize @@ -221,6 +267,45 @@ module ValidationExamples end end + class LazarLastHamsterCrossvalidation < LazarHamsterCrossvalidation + def initialize + super + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/last") + end + end + + ######################################################################################################## + + class LazarHamsterMiniCrossvalidation < CrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + @dataset_file = File.new("data/hamster_carcinogenicity.mini.csv","r") + @num_folds = 2 + end + end + + class ISSCANStratifiedCrossvalidation < CrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + @dataset_file = File.new("data/ISSCAN_v3a_canc-red.csv","r") + @stratified = true + @num_folds = 10 + end + end + + class ISSCAN2StratifiedCrossvalidation < CrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + @dataset_file = File.new("data/ISSCAN_v3a_sal.csv","r") + @stratified = true + @num_folds = 10 + end + end + + ######################################################################################################## class ISTHamsterCrossvalidation < CrossValidation @@ -368,6 +453,74 @@ module ValidationExamples end end + class NtuaModel2 < ModelValidation + def initialize + @model_uri = "http://opentox.ntua.gr:8080/model/11093fbc-3b8b-41e2-bfe3-d83f5f529efc" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/54" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/579820" + end + end + + class NtuaModel3 < ModelValidation + def initialize + @model_uri = "http://opentox.ntua.gr:8080/model/bbab3714-e90b-4990-bef9-8e7d3a30eece" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + #@prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/579820" + end + end + + ######################################################################################################## + + class NtuaTrainingTest < TrainingTestValidation + def initialize + @algorithm_uri = "http://opentox.ntua.gr:8080/algorithm/mlr" + @training_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + end + end + + class NtuaTrainingTestSplit < SplitTestValidation + def initialize + @algorithm_uri = "http://opentox.ntua.gr:8080/algorithm/mlr" + @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + end + end + + class NtuaCrossvalidation < CrossValidation + def initialize + @algorithm_uri = "http://opentox.ntua.gr:8080/algorithm/mlr" + @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + end + end + + class AmbitVsNtuaTrainingTest < TrainingTestValidation + def initialize + @algorithm_uri = "http://apps.ideaconsult.net:8080/ambit2/algorithm/LR" + @training_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + end + end + + + class LazarVsNtuaCrossvalidation < CrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + @num_folds=3 + end + end + + +# loading prediciton via test-dataset:'http://apps.ideaconsult.net:8080/ambit2/dataset/R545', +# test-target-datset:'', prediction-dataset:'http://apps.ideaconsult.net:8080/ambit2/dataset/584389', +# prediction_feature: 'http://apps.ideaconsult.net:8080/ambit2/feature/22200' ', predicted_variable: 'http://apps.ideaconsult.net:8080/ambit2/feature/627667' :: /ot_predictions.rb:21:in `initialize' +#D, [2011-05-11T13:47:26.631628 #22952] DEBUG -- : validation :: ######################################################################################################## class TumModel < ModelValidation @@ -402,6 +555,23 @@ module ValidationExamples end end + class AmbitXYModelValidation < ModelValidation + def initialize + @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/237692" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R736156" + @prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/430905" + end + end + + class AmbitXYZModelValidation < ModelValidation + def initialize + @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/238008" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R736396" + #@prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/430905" ?? + end + end + + class AmbitTrainingTest < TrainingTestValidation def initialize @training_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401560" @@ -528,20 +698,36 @@ module ValidationExamples end end + ######################################################################################################## + + class TumCrossValidation < CrossValidation + def initialize + @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/124963" + @algorithm_uri = "http://opentox:8080/OpenTox/algorithm/kNNregression" + @prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/121905" + @num_folds=2 + super + end + end + ######################################################################################################## @@list = { - "1" => [ LazarHamsterSplit, MajorityHamsterSplit ], + "1" => [ LazarHamsterSplit, MajorityHamsterSplit, MajorityRandomHamsterSplit ], "1a" => [ LazarHamsterSplit ], "1b" => [ MajorityHamsterSplit ], + "1c" => [ MajorityRandomHamsterSplit ], + "1d" => [ LazarLastHamsterSplit ], "2" => [ LazarHamsterTrainingTest, MajorityHamsterTrainingTest ], "2a" => [ LazarHamsterTrainingTest ], "2b" => [ MajorityHamsterTrainingTest ], - "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation ], + "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation, MajorityRandomHamsterCrossvalidation ], "3a" => [ LazarHamsterCrossvalidation ], "3b" => [ MajorityHamsterCrossvalidation ], + "3c" => [ MajorityRandomHamsterCrossvalidation ], + "3d" => [ LazarLastHamsterCrossvalidation ], "4" => [ MajorityISTHamsterCrossvalidation, LazarISTHamsterCrossvalidation, ISTLazarISTHamsterCrossvalidation ], "4a" => [ MajorityISTHamsterCrossvalidation ], @@ -574,11 +760,17 @@ module ValidationExamples "13a" => [ LazarEPAFHMSplit ], "13b" => [ MajorityEPAFHMSplit ], + "13c" => [ MajorityRandomEPAFHMSplit ], + "13d" => [ LazarLastEPAFHMSplit ], + "14" => [ LazarEPAFHMCrossvalidation, MajorityEPAFHMCrossvalidation, MajorityRandomEPAFHMCrossvalidation ], "14a" => [ LazarEPAFHMCrossvalidation ], "14b" => [ MajorityEPAFHMCrossvalidation ], + "14c" => [ MajorityRandomEPAFHMCrossvalidation ], "15a" => [ NtuaModel ], + "15b" => [ NtuaModel2 ], + "15c" => [ NtuaModel3 ], "16" => [ LazarRepdoseSplit, MajorityRepdoseSplit ], "16a" => [ LazarRepdoseSplit ], @@ -599,7 +791,19 @@ module ValidationExamples "19g" => [ AmbitJ48TrainingTest ], "19h" => [ AmbitJ48TrainingTestSplit ], "19i" => [ AmbitAquaticModelValidation ], + "19j" => [ AmbitXYModelValidation ], + + "20a" => [ TumCrossValidation ], + + "21a" => [ LazarHamsterMiniCrossvalidation ], + "21b" => [ ISSCANStratifiedCrossvalidation ], + "21c" => [ ISSCAN2StratifiedCrossvalidation ], + "22a" => [ NtuaTrainingTest ], + "22b" => [ NtuaTrainingTestSplit ], + "22c" => [ NtuaCrossvalidation ], + "22d" => [ LazarVsNtuaCrossvalidation ], + "22e" => [ AmbitVsNtuaTrainingTest ], } diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 99d8672..520d84c 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -63,13 +63,13 @@ module Validation :percent_unpredicted => prediction.percent_unpredicted, :finished => true (VAL_PROPS_GENERAL-[:validation_uri]).each do |p| - v.send("#{p.to_s}=".to_sym, vals.collect{ |vv| vv.send(p) }.uniq.join(",")) + v.send("#{p.to_s}=".to_sym, vals.collect{ |vv| vv.send(p) }.uniq.join(";")) end v.date = crossvalidation.date v.validation_type = "crossvalidation_statistics" v.crossvalidation_id = crossvalidation.id - v.crossvalidation_fold = vals.collect{ |vv| vv.crossvalidation_fold }.uniq.join(",") - v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(",") + v.crossvalidation_fold = vals.collect{ |vv| vv.crossvalidation_fold }.uniq.join(";") + v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(";") v.save end v diff --git a/validation/validation_test.rb b/validation/validation_test.rb index efa8ad5..2c86548 100755 --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -8,7 +8,7 @@ before { require "uri" require "yaml" -ENV['RACK_ENV'] = 'test' +ENV['RACK_ENV'] = 'production' require 'application.rb' require 'test/unit' require 'rack/test' @@ -20,10 +20,10 @@ LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S " LOGGER.formatter = Logger::Formatter.new if AA_SERVER - TEST_USER = "mgtest" - TEST_PW = "mgpasswd" - #TEST_USER = "guest" - #TEST_PW = "guest" + #TEST_USER = "mgtest" + #TEST_PW = "mgpasswd" + TEST_USER = "guest" + TEST_PW = "guest" SUBJECTID = OpenTox::Authorization.authenticate(TEST_USER,TEST_PW) raise "could not log in" unless SUBJECTID puts "logged in: "+SUBJECTID.to_s @@ -60,6 +60,171 @@ class ValidationTest < Test::Unit::TestCase begin $test_case = self + #get 'crossvalidation/138/statistics' + #get 'crossvalidation/189/statistics' + #puts last_response.body +# run_test("1b") + + #get '/crossvalidation/79/predictions',nil,'HTTP_ACCEPT' => "application/x-yaml" + #puts last_response.body + + #run_test("22e") #,:validation_uri=>"http://local-ot/validation/84" ) + #run_test("21b") + #run_test("21c") + + # get '?media=text/uri-list' + + #post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/135,http://local-ot/validation/crossvalidation/134"} + #post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/174,http://local-ot/validation/crossvalidation/175"} + # 2 majority, 175 is real maj, 176 is random + +# post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/185,http://local-ot/validation/crossvalidation/193,http://local-ot/validation/crossvalidation/186,http://local-ot/validation/crossvalidation/194,http://local-ot/validation/crossvalidation/187,http://local-ot/validation/crossvalidation/195", +# :identifier=>"lazar,lazar,real_majority,real_majority,random_classification,random_classification"} +# uri = last_response.body +# rep = wait_for_task(uri) +# puts rep + +# post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/199,http://local-ot/validation/crossvalidation/204,http://local-ot/validation/crossvalidation/203", +# :identifier=>"lazar,real_majority,random_classification"} +# uri = last_response.body +# rep = wait_for_task(uri) +# puts rep + # 205 206 207 + +# post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/149,http://local-ot/validation/crossvalidation/210", +# :identifier=>"bbrc,last"} +# uri = last_response.body +# rep = wait_for_task(uri) +# puts rep + + #run_test("1a", {:validation_uri=>"http://local-ot/validation/466"}) +# puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + #run_test("3a",{:validation_uri=>"http://local-ot/validation/crossvalidation/149"}) + #puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + #run_test("13a", {:validation_uri=>"http://local-ot/validation/406"}) +# puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + #run_test("14a",{:validation_uri=>"http://local-ot/validation/crossvalidation/148"}) +# puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + + run_test("1a") + +# run_test("3d",{ +# :dataset_uri => "http://local-ot/dataset/2897", +# :prediction_feature => "http://local-ot/dataset/2897/feature/Hamster%20Carcinogenicity", +# :random_seed => 1 +# }) + + #run_test("14",{ + # :dataset_uri => "http://local-ot/dataset/3877", + # :prediction_feature => "http://local-ot/dataset/3877/feature/LC50_mmol", + # :random_seed => 2 + # }) + #puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + +# get "?model=http://local-ot/model/330" +# puts last_response.body +# puts "\n\n" +# get "" +# puts last_response.body + + #get "report/validation?validation=http://local-ot/validation/167" + #puts last_response.body + +# run_test("3a") #,:validation_uri=>"http://local-ot/validation/84" ) + #get "report/crossvalidation?crossvalidation=http://local-ot/validation/crossvalidation/47" + #puts last_response.body + + + rescue => ex + rep = OpenTox::ErrorReport.create(ex, "") + puts rep.to_yaml + ensure + #OpenTox::Authorization.logout(SUBJECTID) if AA_SERVER + end + end + + def app + Sinatra::Application + end + + def run_test(select=nil, overwrite={}, delete=false ) + + if AA_SERVER && SUBJECTID && delete + policies_before = OpenTox::Authorization.list_policy_uris(SUBJECTID) + end + + puts ValidationExamples.list unless select + validationExamples = ValidationExamples.select(select) + validationExamples.each do |vv| + vv.each do |v| + ex = v.new + ex.subjectid = SUBJECTID + + overwrite.each do |k,v| + ex.send(k.to_s+"=",v) + end + + unless ex.validation_uri + ex.upload_files + ex.check_requirements + ex.validate + + LOGGER.debug "validation done '"+ex.validation_uri.to_s+"'" + end + if !delete and ex.validation_uri + if SUBJECTID + puts ex.validation_uri+"?subjectid="+CGI.escape(SUBJECTID) + else + puts ex.validation_uri + end + end + + unless ex.report_uri + ex.report + end + if !delete and ex.report_uri + if SUBJECTID + puts ex.report_uri+"?subjectid="+CGI.escape(SUBJECTID) + else + puts ex.report_uri + end + end + ##ex.verify_yaml + ##ex.compare_yaml_vs_rdf + ex.delete if delete + end + end + + if AA_SERVER && SUBJECTID && delete + policies_after= OpenTox::Authorization.list_policy_uris(SUBJECTID) + diff = policies_after.size - policies_before.size + if (diff != 0) + policies_before.each do |k,v| + policies_after.delete(k) + end + LOGGER.warn diff.to_s+" policies NOT deleted:\n"+policies_after.collect{|k,v| k.to_s+" => "+v.to_s}.join("\n") + else + LOGGER.debug "all policies deleted" + end + end + end + + def prepare_examples + get '/prepare_examples' + end + + def do_test_examples # USES CURL, DO NOT FORGET TO RESTART + post '/test_examples' + end + + def do_test_examples_ortona + post '/test_examples',:examples=>"http://ortona.informatik.uni-freiburg.de/validation/examples" + end + +end + + + # prediction_feature = "https://ambit.uni-plovdiv.bg:8443/ambit2/feature/26221" # puts OpenTox::Feature.find(prediction_feature).domain.inspect # exit @@ -137,6 +302,18 @@ class ValidationTest < Test::Unit::TestCase # :regression=>"true"} # #:classification=>"true"} # puts last_response.body + +# post "/validate_datasets",{ +# :test_dataset_uri=>"http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=10", +# :prediction_dataset_uri=>"http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=10", +# #:test_target_dataset_uri=>"http://local-ot/dataset/202", +# :prediction_feature=>"http://apps.ideaconsult.net:8080/ambit2/feature/21573", +# :predicted_feature=>"http://apps.ideaconsult.net:8080/ambit2/feature/21573", +# #:regression=>"true"} +# :classification=>"true"} +# puts last_response.body + + #run_test("1a") #,:validation_uri=>"http://local-ot/validation/84" ) # post "/validate_datasets",{ # :test_dataset_uri=>"http://local-ot/dataset/89", @@ -251,13 +428,17 @@ class ValidationTest < Test::Unit::TestCase #puts "" #puts last_response.body #exit + +# run_test("20a") # get "/error" # puts last_response.body #delete "/1",:subjectid=>SUBJECTID - prepare_examples() + #prepare_examples() + + #run_test("15b") #run_test("1a") #,{:validation_uri => "http://local-ot/validation/crossvalidation/1"}) @@ -275,92 +456,4 @@ class ValidationTest < Test::Unit::TestCase #prepare_examples #do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE - #do_test_examples_ortona - - rescue => ex - rep = OpenTox::ErrorReport.create(ex, "") - puts rep.to_yaml - ensure - #OpenTox::Authorization.logout(SUBJECTID) if AA_SERVER - end - end - - def app - Sinatra::Application - end - - def run_test(select=nil, overwrite={}, delete=false ) - - if AA_SERVER && SUBJECTID && delete - policies_before = OpenTox::Authorization.list_policy_uris(SUBJECTID) - end - - puts ValidationExamples.list unless select - validationExamples = ValidationExamples.select(select) - validationExamples.each do |vv| - vv.each do |v| - ex = v.new - ex.subjectid = SUBJECTID - - overwrite.each do |k,v| - ex.send(k.to_s+"=",v) - end - - unless ex.validation_uri - ex.upload_files - ex.check_requirements - ex.validate - - LOGGER.debug "validation done '"+ex.validation_uri.to_s+"'" - end - if !delete and ex.validation_uri - if SUBJECTID - puts ex.validation_uri+"?subjectid="+CGI.escape(SUBJECTID) - else - puts ex.validation_uri - end - end - - unless ex.report_uri - ex.report - end - if !delete and ex.report_uri - if SUBJECTID - puts ex.report_uri+"?subjectid="+CGI.escape(SUBJECTID) - else - puts ex.report_uri - end - end - ##ex.verify_yaml - ##ex.compare_yaml_vs_rdf - ex.delete if delete - end - end - - if AA_SERVER && SUBJECTID && delete - policies_after= OpenTox::Authorization.list_policy_uris(SUBJECTID) - diff = policies_after.size - policies_before.size - if (diff != 0) - policies_before.each do |k,v| - policies_after.delete(k) - end - LOGGER.warn diff.to_s+" policies NOT deleted:\n"+policies_after.collect{|k,v| k.to_s+" => "+v.to_s}.join("\n") - else - LOGGER.debug "all policies deleted" - end - end - end - - def prepare_examples - get '/prepare_examples' - end - - def do_test_examples # USES CURL, DO NOT FORGET TO RESTART - post '/test_examples' - end - - def do_test_examples_ortona - post '/test_examples',:examples=>"http://ortona.informatik.uni-freiburg.de/validation/examples" - end - -end + #do_test_examples_ortona
\ No newline at end of file |