From 9ce03c0f50bb9129b584327d56fa4c9277849227 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 17 May 2011 10:46:45 +0200 Subject: crossvalidation statistics fix: compute cv-statistics with cv-predictions instead of averaging cv-validation-statistics --- lib/dataset_cache.rb | 23 ++++ lib/ot_predictions.rb | 254 ++++++++++++++++++++--------------- reach_reports/reach_service.rb | 47 ++++--- report/plot_factory.rb | 2 +- report/report_content.rb | 14 +- report/validation_access.rb | 32 ++++- report/validation_data.rb | 39 ++++-- validation/validation_application.rb | 35 +---- validation/validation_format.rb | 2 +- validation/validation_service.rb | 84 +++++++----- 10 files changed, 322 insertions(+), 210 deletions(-) create mode 100644 lib/dataset_cache.rb diff --git a/lib/dataset_cache.rb b/lib/dataset_cache.rb new file mode 100644 index 0000000..1af1d51 --- /dev/null +++ b/lib/dataset_cache.rb @@ -0,0 +1,23 @@ + +module Lib + + module DatasetCache + + @@cache={} + + # same as OpenTox::Dataset.find with caching function + # rational: datasets are reused in crossvalidation very often, cache to save computational effort + # PENDING: may cause memory issues, test with huge datasets + def self.find(dataset_uri, subjectid=nil) + return nil if (dataset_uri==nil) + d = @@cache[dataset_uri.to_s+"_"+subjectid.to_s] + if d==nil + d = OpenTox::Dataset.find(dataset_uri, subjectid) + @@cache[dataset_uri.to_s+"_"+subjectid.to_s] = d + end + d + end + + end + +end \ No newline at end of file diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index 22f9b20..644168f 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -15,127 +15,161 @@ module Lib return @compounds[instance_index] end - def initialize(feature_type, test_dataset_uri, test_target_dataset_uri, - prediction_feature, prediction_dataset_uri, predicted_variable, subjectid=nil, task=nil) + def initialize( feature_type, test_dataset_uris, test_target_dataset_uris, + prediction_feature, prediction_dataset_uris, predicted_variables, subjectid=nil, task=nil) - LOGGER.debug("loading prediciton via test-dataset:'"+test_dataset_uri.to_s+ - "', test-target-datset:'"+test_target_dataset_uri.to_s+ - "', prediction-dataset:'"+prediction_dataset_uri.to_s+ - "', prediction_feature: '"+prediction_feature.to_s+"' "+ - "', predicted_variable: '"+predicted_variable.to_s+"'") - - predicted_variable=prediction_feature if predicted_variable==nil - - test_dataset = OpenTox::Dataset.find test_dataset_uri,subjectid - raise "test dataset not found: '"+test_dataset_uri.to_s+"'" unless test_dataset + test_dataset_uris = [test_dataset_uris] unless test_dataset_uris.is_a?(Array) + test_target_dataset_uris = [test_target_dataset_uris] unless test_target_dataset_uris.is_a?(Array) + prediction_dataset_uris = [prediction_dataset_uris] unless prediction_dataset_uris.is_a?(Array) + predicted_variables = [predicted_variables] unless predicted_variables.is_a?(Array) + LOGGER.debug "loading prediciton -- test-dataset: "+test_dataset_uris.inspect + LOGGER.debug "loading prediciton -- test-target-datset: "+test_target_dataset_uris.inspect + LOGGER.debug "loading prediciton -- prediction-dataset: "+prediction_dataset_uris.inspect + LOGGER.debug "loading prediciton -- predicted_variable: "+predicted_variables.inspect + LOGGER.debug "loading prediciton -- prediction_feature: "+prediction_feature.to_s raise "prediction_feature missing" unless prediction_feature - if test_target_dataset_uri == nil || test_target_dataset_uri.strip.size==0 || test_target_dataset_uri==test_dataset_uri - test_target_dataset_uri = test_dataset_uri - test_target_dataset = test_dataset - raise "prediction_feature not found in test_dataset, specify a test_target_dataset\n"+ - "prediction_feature: '"+prediction_feature.to_s+"'\n"+ - "test_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ - "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil - else - test_target_dataset = OpenTox::Dataset.find test_target_dataset_uri,subjectid - raise "test target datset not found: '"+test_target_dataset_uri.to_s+"'" unless test_target_dataset - if CHECK_VALUES - test_dataset.compounds.each do |c| - raise "test compound not found on test class dataset "+c.to_s unless test_target_dataset.compounds.include?(c) - end - end - raise "prediction_feature not found in test_target_dataset\n"+ - "prediction_feature: '"+prediction_feature.to_s+"'\n"+ - "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ - "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil - end - - @compounds = test_dataset.compounds - LOGGER.debug "test dataset size: "+@compounds.size.to_s - raise "test dataset is empty "+test_dataset_uri.to_s unless @compounds.size>0 - - if feature_type=="classification" - accept_values = test_target_dataset.features[prediction_feature][OT.acceptValue] - raise "'"+OT.acceptValue.to_s+"' missing/invalid for feature '"+prediction_feature.to_s+"' in dataset '"+ - test_target_dataset_uri.to_s+"', acceptValues are: '"+accept_values.inspect+"'" if accept_values==nil or accept_values.length<2 - else - accept_values=nil - end + @compounds = [] + all_predicted_values = [] + all_actual_values = [] + all_confidence_values = [] + accept_values = nil - actual_values = [] - @compounds.each do |c| - case feature_type - when "classification" - actual_values << classification_value(test_target_dataset, c, prediction_feature, accept_values) - when "regression" - actual_values << regression_value(test_target_dataset, c, prediction_feature) - end + if task + task_step = 100 / (test_dataset_uris.size*2 + 1) + task_status = 0 end - task.progress(40) if task # loaded actual values + + test_dataset_uris.size.times do |i| + + test_dataset_uri = test_dataset_uris[i] + test_target_dataset_uri = test_target_dataset_uris[i] + prediction_dataset_uri = prediction_dataset_uris[i] + predicted_variable = predicted_variables[i] + + predicted_variable=prediction_feature if predicted_variable==nil - prediction_dataset = OpenTox::Dataset.find prediction_dataset_uri,subjectid - raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset + test_dataset = Lib::DatasetCache.find test_dataset_uri,subjectid + raise "test dataset not found: '"+test_dataset_uri.to_s+"'" unless test_dataset - # TODO: remove LAZAR_PREDICTION_DATASET_HACK - no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil - if no_prediction_feature - one_entry_per_compound = true - @compounds.each do |c| - if prediction_dataset.data_entries[c] and prediction_dataset.data_entries[c].size != 1 - one_entry_per_compound = false - break - end - end - msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+ - prediction_dataset.features.keys.inspect - if one_entry_per_compound - LOGGER.warn msg + if test_target_dataset_uri == nil || test_target_dataset_uri.strip.size==0 || test_target_dataset_uri==test_dataset_uri + test_target_dataset_uri = test_dataset_uri + test_target_dataset = test_dataset + raise "prediction_feature not found in test_dataset, specify a test_target_dataset\n"+ + "prediction_feature: '"+prediction_feature.to_s+"'\n"+ + "test_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ + "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil else - raise msg + test_target_dataset = Lib::DatasetCache.find test_target_dataset_uri,subjectid + raise "test target datset not found: '"+test_target_dataset_uri.to_s+"'" unless test_target_dataset + if CHECK_VALUES + test_dataset.compounds.each do |c| + raise "test compound not found on test class dataset "+c.to_s unless test_target_dataset.compounds.include?(c) + end + end + raise "prediction_feature not found in test_target_dataset\n"+ + "prediction_feature: '"+prediction_feature.to_s+"'\n"+ + "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ + "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil end - end - - raise "more predicted than test compounds test:"+@compounds.size.to_s+" < prediction:"+ - prediction_dataset.compounds.size.to_s if @compounds.size < prediction_dataset.compounds.size - if CHECK_VALUES - prediction_dataset.compounds.each do |c| - raise "predicted compound not found in test dataset:\n"+c+"\ntest-compounds:\n"+ - @compounds.collect{|c| c.to_s}.join("\n") if @compounds.index(c)==nil + + compounds = test_dataset.compounds + LOGGER.debug "test dataset size: "+compounds.size.to_s + raise "test dataset is empty "+test_dataset_uri.to_s unless compounds.size>0 + + if feature_type=="classification" + av = test_target_dataset.features[prediction_feature][OT.acceptValue] + raise "'"+OT.acceptValue.to_s+"' missing/invalid for feature '"+prediction_feature.to_s+"' in dataset '"+ + test_target_dataset_uri.to_s+"', acceptValues are: '"+av.inspect+"'" if av==nil or av.length<2 + if accept_values==nil + accept_values=av + else + raise "accept values (in folds) differ "+av.inspect+" != "+accept_values.inspect if av!=accept_values + end end - end - - predicted_values = [] - confidence_values = [] - @compounds.each do |c| - if prediction_dataset.compounds.index(c)==nil - predicted_values << nil - confidence_values << nil - else + + actual_values = [] + compounds.each do |c| case feature_type when "classification" - # TODO: remove LAZAR_PREDICTION_DATASET_HACK - predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, accept_values) + actual_values << classification_value(test_target_dataset, c, prediction_feature, accept_values) when "regression" - predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable) + actual_values << regression_value(test_target_dataset, c, prediction_feature) + end + end + task.progress( task_status += task_step ) if task # loaded actual values + + prediction_dataset = Lib::DatasetCache.find prediction_dataset_uri,subjectid + raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset + + # TODO: remove LAZAR_PREDICTION_DATASET_HACK + no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil + if no_prediction_feature + one_entry_per_compound = true + compounds.each do |c| + if prediction_dataset.data_entries[c] and prediction_dataset.data_entries[c].size != 1 + one_entry_per_compound = false + break + end + end + msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+ + prediction_dataset.features.keys.inspect + if one_entry_per_compound + LOGGER.warn msg + else + raise msg end - # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable) - conf = 1 - begin - feature = prediction_dataset.data_entries[c].keys[0] - feature_data = prediction_dataset.features[feature] - conf = feature_data[OT.confidence] if feature_data[OT.confidence]!=nil - rescue - LOGGER.warn "could not get confidence" + end + + raise "more predicted than test compounds test:"+compounds.size.to_s+" < prediction:"+ + prediction_dataset.compounds.size.to_s if compounds.size < prediction_dataset.compounds.size + if CHECK_VALUES + prediction_dataset.compounds.each do |c| + raise "predicted compound not found in test dataset:\n"+c+"\ntest-compounds:\n"+ + compounds.collect{|c| c.to_s}.join("\n") if compounds.index(c)==nil end - confidence_values << conf end + + predicted_values = [] + confidence_values = [] + count = 0 + compounds.each do |c| + if prediction_dataset.compounds.index(c)==nil + predicted_values << nil + confidence_values << nil + else + case feature_type + when "classification" + # TODO: remove LAZAR_PREDICTION_DATASET_HACK + predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, accept_values) + when "regression" + predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable) + end + # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable) + conf = predicted_values[count]!=nil ? 1 : 0 + begin + feature = prediction_dataset.data_entries[c].keys[0] + feature_data = prediction_dataset.features[feature] + conf = feature_data[OT.confidence] if feature_data[OT.confidence]!=nil + rescue + LOGGER.warn "could not get confidence" + end + confidence_values << conf + end + count += 1 + end + + @compounds += compounds + all_predicted_values += predicted_values + all_actual_values += actual_values + all_confidence_values += confidence_values + + task.progress( task_status += task_step ) if task # loaded predicted values and confidence end - task.progress(80) if task # loaded predicted values and confidence - super(predicted_values, actual_values, confidence_values, feature_type, accept_values) - raise "illegal num compounds "+num_info if @compounds.size != @predicted_values.size - task.progress(100) if task # done with the mathmatics + super(all_predicted_values, all_actual_values, all_confidence_values, feature_type, accept_values) + raise "illegal num compounds "+num_info if @compounds.size != @predicted_values.size + task.progress(100) if task # done with the mathmatics end private @@ -205,6 +239,7 @@ module Lib def self.to_array( predictions, add_pic=false, format=false ) res = [] + conf_column = nil predictions.each do |p| (0..p.num_instances-1).each do |i| a = [] @@ -230,13 +265,22 @@ module Lib end end if p.confidence_values_available? - a << (format ? p.confidence_value(i).to_nice_s : p.confidence_value(i)) + conf_column = a.size + a << p.confidence_value(i) #(format ? p.confidence_value(i).to_nice_s : p.confidence_value(i)) end a << p.identifier(i) res << a end end - + + if conf_column!=nil + res.sort!{ |x,y| y[4] <=> x[4] } + if format + res.each do |a| + a[4] = a[4].to_nice_s + end + end + end header = [] header << "compound" if add_pic header << "actual value" diff --git a/reach_reports/reach_service.rb b/reach_reports/reach_service.rb index 0cf4172..b6c6350 100755 --- a/reach_reports/reach_service.rb +++ b/reach_reports/reach_service.rb @@ -162,33 +162,36 @@ module ReachReports next if cvs.size==0 lmo << "crossvalidation/s on "+desc cvs.each do |cv| - lmo << "crossvalidation: "+cv.crossvalidation_uri - lmo << "dataset (see 9.3 Validation data): "+cv.dataset_uri - val_datasets << cv.dataset_uri - lmo << "settings: num-folds="+cv.num_folds.to_s+", random-seed="+cv.random_seed.to_s+", stratified:"+cv.stratified.to_s - - val = YAML.load( OpenTox::RestClientWrapper.get(File.join(cv.crossvalidation_uri,"statistics"),{:subjectid => r.subjectid}) ) - case feature_type - when "classification" - lmo << "percent_correct: "+val[OT.classificationStatistics][OT.percentCorrect].to_s - lmo << "weighted AUC: "+val[OT.classificationStatistics][OT.weightedAreaUnderRoc].to_s - when "regression" - lmo << "root_mean_squared_error: "+val[OT.regressionStatistics][OT.rootMeanSquaredError].to_s - lmo << "r_square "+val[OT.regressionStatistics][OT.rSquare].to_s - end - reports = OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"], - "report/crossvalidation?crossvalidation_uris="+cv.crossvalidation_uri),{:subjectid => r.subjectid}) - if reports and reports.chomp.size>0 - lmo << "for more info see report: "+reports.split("\n")[0] - else - lmo << "for more info see report: not yet created for '"+cv.crossvalidation_uri+"'" + begin + lmo << "crossvalidation: "+cv.crossvalidation_uri + lmo << "dataset (see 9.3 Validation data): "+cv.dataset_uri + val_datasets << cv.dataset_uri + lmo << "settings: num-folds="+cv.num_folds.to_s+", random-seed="+cv.random_seed.to_s+", stratified:"+cv.stratified.to_s + + val = YAML.load( OpenTox::RestClientWrapper.get(File.join(cv.crossvalidation_uri,"statistics"),{:subjectid => r.subjectid}) ) + case feature_type + when "classification" + lmo << "percent_correct: "+val[OT.classificationStatistics][OT.percentCorrect].to_s + lmo << "weighted AUC: "+val[OT.classificationStatistics][OT.weightedAreaUnderRoc].to_s + when "regression" + lmo << "root_mean_squared_error: "+val[OT.regressionStatistics][OT.rootMeanSquaredError].to_s + lmo << "r_square "+val[OT.regressionStatistics][OT.rSquare].to_s + end + reports = OpenTox::RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"], + "report/crossvalidation?crossvalidation_uris="+cv.crossvalidation_uri),{:subjectid => r.subjectid}) + if reports and reports.chomp.size>0 + lmo << "for more info see report: "+reports.split("\n")[0] + else + lmo << "for more info see report: not yet created for '"+cv.crossvalidation_uri+"'" + end + rescue => ex + LOGGER.warn "could not add cv "+cv.crossvalidation_uri+" : "+ex.message end end lmo << "" end end - - else + else lmo = [ "no prediction algortihm for model found, crossvalidation not possible" ] end r.qsar_robustness.lmo = lmo.to_html diff --git a/report/plot_factory.rb b/report/plot_factory.rb index a4e415a..b7c920a 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -354,7 +354,7 @@ module Reports c = roc_values[:confidence_values] p = roc_values[:predicted_values] a = roc_values[:actual_values] - raise "no prediction values for roc-plot" if p.size==0 + raise "no prediction values for confidence plot" if p.size==0 (0..p.size-2).each do |i| ((i+1)..p.size-1).each do |j| diff --git a/report/report_content.rb b/report/report_content.rb index ca04f25..cc4c13c 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -31,11 +31,11 @@ class Reports::ReportContent level = 0.90 test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, group_attribute, test_attribute, "paired_ttest", level ) - puts test_matrix.inspect + #puts test_matrix.inspect titles = test_matrix[:titles] matrix = test_matrix[:matrix] table = [] - puts titles.inspect + #puts titles.inspect table << [""] + titles titles.size.times do |i| table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") } @@ -47,10 +47,10 @@ class Reports::ReportContent end def add_predictions( validation_set, - validation_attributes=[], - section_title="Predictions", - section_text=nil, - table_title="Predictions") + validation_attributes=[], + section_title="Predictions", + section_text=nil, + table_title="Predictions") #PENING raise "validation attributes not implemented in get prediction array" if validation_attributes.size>0 @@ -109,7 +109,7 @@ class Reports::ReportContent if (search_for_existing_report_type) vals.size.times do |i| - puts i + #puts i if (i==0) vals[i] = [ "Reports" ] + vals[i] puts vals[i].inspect diff --git a/report/validation_access.rb b/report/validation_access.rb index e9b6e19..d0c3a1d 100755 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -25,7 +25,7 @@ class Reports::ValidationDB raise OpenTox::NotFoundError.new "crossvalidation with id "+cv_id.to_s+" not found" unless cv raise OpenTox::BadRequestError.new("crossvalidation with id '"+cv_id.to_s+"' not finished") unless cv.finished #res += Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s} - res += Validation::Validation.find( :crossvalidation_id => cv_id ).collect{|v| v.validation_uri.to_s } + res += Validation::Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|v| v.validation_uri.to_s } else res += [u.to_s] end @@ -35,7 +35,7 @@ class Reports::ValidationDB def init_validation(validation, uri, subjectid=nil) - raise OpenTox::BadRequestError.new "not a validation uri: "+uri.to_s unless uri =~ /.*\/[0-9]+/ + raise OpenTox::BadRequestError.new "not a validation uri: "+uri.to_s unless uri =~ /\/[0-9]+$/ validation_id = uri.split("/")[-1] raise OpenTox::BadRequestError.new "invalid validation id "+validation_id.to_s unless validation_id!=nil and (validation_id.to_i > 0 || validation_id.to_s=="0" ) @@ -56,6 +56,31 @@ class Reports::ValidationDB subset_props.each{ |prop| validation.send("#{prop.to_s}=".to_sym, subset[prop]) } if subset end end + + def init_validation_from_cv_statistics( validation, cv_uri, subjectid=nil ) + + raise OpenTox::BadRequestError.new "not a crossvalidation uri: "+cv_uri.to_s unless cv_uri.uri? and cv_uri =~ /crossvalidation.*\/[0-9]+$/ + cv_id = cv_uri.split("/")[-1] + raise OpenTox::NotAuthorizedError.new "Not authorized: GET "+cv_uri.to_s if + AA_SERVER and !OpenTox::Authorization.authorized?(cv_uri,"GET",subjectid) + cv = Validation::Crossvalidation.get(cv_id) + raise OpenTox::NotFoundError.new "crossvalidation with id "+crossvalidation_id.to_s+" not found" unless cv + raise OpenTox::BadRequestError.new "crossvalidation with id "+crossvalidation_id.to_s+" is not finished yet" unless cv.finished + v = Validation::Validation.from_cv_statistics(cv_id, subjectid) + (Validation::VAL_PROPS + Validation::VAL_CV_PROPS).each do |p| + validation.send("#{p.to_s}=".to_sym, v.send(p)) + end + {:classification_statistics => Validation::VAL_CLASS_PROPS, + :regression_statistics => Validation::VAL_REGR_PROPS}.each do |subset_name,subset_props| + subset = v.send(subset_name) + subset_props.each{ |prop| validation.send("#{prop.to_s}=".to_sym, subset[prop]) } if subset + end + #cv props + Validation::CROSS_VAL_PROPS.each do |p| + validation.send("#{p.to_s}=".to_sym, cv.send(p.to_s)) + end + validation.crossvalidation_uri = cv_uri + end def init_cv(validation) @@ -76,7 +101,8 @@ class Reports::ValidationDB def get_accept_values( validation, subjectid=nil ) # PENDING So far, one has to load the whole dataset to get the accept_value from ambit - d = OpenTox::Dataset.find( validation.test_target_dataset_uri, subjectid ) + d = Lib::DatasetCache.find( validation.test_target_dataset_uri, subjectid ) + raise "cannot get test target dataset for accept values, dataset: "+validation.test_target_dataset_uri.to_s unless d accept_values = d.features[validation.prediction_feature][OT.acceptValue] raise "cannot get accept values from dataset "+validation.test_target_dataset_uri.to_s+" for feature "+ validation.prediction_feature+":\n"+d.features[validation.prediction_feature].to_yaml unless accept_values!=nil diff --git a/report/validation_data.rb b/report/validation_data.rb index 42b179b..11fa737 100755 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -81,6 +81,12 @@ module Reports @subjectid = subjectid #raise "subjectid is nil" unless subjectid end + + def self.from_cv_statistics( cv_uri, subjectid = nil ) + v = ReportValidation.new(nil, subjectid) + @@validation_access.init_validation_from_cv_statistics(v, cv_uri, subjectid) + v + end # returns/creates predictions, cache to save rest-calls/computation time # @@ -409,17 +415,30 @@ module Reports #compute grouping grouping = Util.group(@validations, equal_attributes) #puts "groups "+grouping.size.to_s - - Lib::MergeObjects.register_merge_attributes( ReportValidation, - Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL) unless - Lib::MergeObjects.merge_attributes_registered?(ReportValidation) - - #merge - grouping.each do |g| - new_set.validations.push(g[0].clone_validation) - g[1..-1].each do |v| - new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v) + + if ( equal_attributes.include?(:crossvalidation_id) ) + # do not merge, use crossvalidation statistics + raise "statistics vs merging problem" if equal_attributes.size!=1 + grouping.each do |g| + new_set.validations << ReportValidation.from_cv_statistics(g[0].crossvalidation_uri) end + else + #merge + Lib::MergeObjects.register_merge_attributes( ReportValidation, + Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL) unless + Lib::MergeObjects.merge_attributes_registered?(ReportValidation) + grouping.each do |g| + new_set.validations << g[0].clone_validation + w = 1 + g[1..-1].each do |v| + new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v,w,1) + w+=1 + end + end + end + + new_set.validations.each do |v| + raise "not a validation "+v.class.to_s+" "+v.to_s unless v.is_a?(Reports::ReportValidation) end return new_set diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 4bcd07d..7db2a6a 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -3,8 +3,7 @@ require lib end -require 'lib/merge.rb' -#require 'lib/active_record_setup.rb' +require 'lib/dataset_cache.rb' require 'validation/validation_service.rb' get '/crossvalidation/?' do @@ -41,6 +40,8 @@ post '/crossvalidation/?' do cv = Validation::Crossvalidation.create cv_params cv.subjectid = @subjectid cv.perform_cv( params[:prediction_feature], params[:algorithm_params], task ) + # computation of stats is cheap as dataset are already loaded into the memory + Validation::Validation.from_cv_statistics( cv.id, @subjectid ) cv.crossvalidation_uri end return_task(task) @@ -108,33 +109,9 @@ get '/crossvalidation/:id' do end get '/crossvalidation/:id/statistics' do - LOGGER.info "get merged validation-result for crossvalidation with id "+params[:id].to_s -# begin - #crossvalidation = Validation::Crossvalidation.find(params[:id]) -# rescue ActiveRecord::RecordNotFound => ex -# raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found." -# end - #crossvalidation = Validation::Crossvalidation.find(params[:id]) - crossvalidation = Validation::Crossvalidation.get(params[:id]) - - raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found." unless crossvalidation - raise OpenTox::BadRequestError.new "Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished - - Lib::MergeObjects.register_merge_attributes( Validation::Validation, - Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL-[:date,:validation_uri,:crossvalidation_uri]) unless - Lib::MergeObjects.merge_attributes_registered?(Validation::Validation) - - #v = Lib::MergeObjects.merge_array_objects( Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) ) - # convert ohm:set into array, as ohm:set[0]=nil(!) - vals = Validation::Validation.find( :crossvalidation_id => params[:id] ).collect{|x| x} -# LOGGER.debug vals.collect{|v| v.validation_uri}.join("\n") -# LOGGER.debug vals.size -# LOGGER.debug vals.class - raise "could not load all validations for crossvalidation" if vals.include?(nil) - v = Lib::MergeObjects.merge_array_objects( vals ) - v.date = nil - #v.id = nil + LOGGER.info "get crossvalidation statistics for crossvalidation with id "+params[:id].to_s + v = Validation::Validation.from_cv_statistics( params[:id], @subjectid ) case request.env['HTTP_ACCEPT'].to_s when /text\/html/ related_links = @@ -187,7 +164,7 @@ get '/crossvalidation/:id/predictions' do raise OpenTox::BadRequestError.new "Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished content_type "application/x-yaml" - validations = Validation::Validation.find( :crossvalidation_id => params[:id] ) + validations = Validation::Validation.find( :crossvalidation_id => params[:id], :validation_type => "crossvalidation" ) p = Lib::OTPredictions.to_array( validations.collect{ |v| v.compute_validation_stats_with_model(nil, true) } ).to_yaml case request.env['HTTP_ACCEPT'].to_s diff --git a/validation/validation_format.rb b/validation/validation_format.rb index 6fdea61..23b1996 100755 --- a/validation/validation_format.rb +++ b/validation/validation_format.rb @@ -83,7 +83,7 @@ module Validation end v = [] #Validation.find( :all, :conditions => { :crossvalidation_id => self.id } ).each do |val| - Validation.find( :crossvalidation_id => self.id ).each do |val| + Validation.find( :crossvalidation_id => self.id, :validation_type => "crossvalidation" ).each do |val| v.push( val.validation_uri.to_s ) end h[:validation_uris] = v diff --git a/validation/validation_service.rb b/validation/validation_service.rb index dcfb8d7..99d8672 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -31,15 +31,49 @@ end module Validation class Validation - - # constructs a validation object, Rsets id und uri - #def initialize( params={} ) - #raise "do not set id manually" if params[:id] - #params[:finished] = false - #super params - #self.save! - #raise "internal error, validation-id not set "+to_yaml if self.id==nil - #end + + def self.from_cv_statistics( cv_id, subjectid=nil ) + v = Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation_statistics" ).first + unless v + crossvalidation = Crossvalidation.get(cv_id) + raise OpenTox::NotFoundError.new "Crossvalidation '#{cv_id}' not found." unless crossvalidation + raise OpenTox::BadRequestError.new "Crossvalidation '"+cv_id.to_s+"' not finished" unless crossvalidation.finished + + vals = Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|x| x} + feature_type = OpenTox::Model::Generic.new(vals.first.model_uri).feature_type(@subjectid) + test_dataset_uris = vals.collect{|v| v.test_dataset_uri} + test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri} + prediction_feature = vals.first.prediction_feature + prediction_dataset_uris = vals.collect{|v| v.prediction_dataset_uri} + predicted_variables = vals.collect{|v| nil} + prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, + prediction_dataset_uris, predicted_variables, @subjectid ) + + v = Validation.new + case feature_type + when "classification" + v.classification_statistics = prediction.compute_stats + when "regression" + v.regression_statistics = prediction.compute_stats + end + v.update :num_instances => prediction.num_instances, + :num_without_class => prediction.num_without_class, + :percent_without_class => prediction.percent_without_class, + :num_unpredicted => prediction.num_unpredicted, + :percent_unpredicted => prediction.percent_unpredicted, + :finished => true + (VAL_PROPS_GENERAL-[:validation_uri]).each do |p| + v.send("#{p.to_s}=".to_sym, vals.collect{ |vv| vv.send(p) }.uniq.join(",")) + end + v.date = crossvalidation.date + v.validation_type = "crossvalidation_statistics" + v.crossvalidation_id = crossvalidation.id + v.crossvalidation_fold = vals.collect{ |vv| vv.crossvalidation_fold }.uniq.join(",") + v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(",") + v.save + end + v + end # deletes a validation # PENDING: model and referenced datasets are deleted as well, keep it that way? @@ -238,21 +272,7 @@ module Validation class Crossvalidation - # constructs a crossvalidation, id and uri are set - #def initialize( params={} ) - # - # raise "do not set id manually" if params[:id] - # params[:num_folds] = 10 if params[:num_folds]==nil - # params[:random_seed] = 1 if params[:random_seed]==nil - # params[:stratified] = false if params[:stratified]==nil - # params[:finished] = false - # super params - # self.save! - # raise "internal error, crossvalidation-id not set" if self.id==nil - #end - def perform_cv ( prediction_feature, algorithm_params=nil, task=nil ) - create_cv_datasets( prediction_feature, OpenTox::SubTask.create(task, 0, 33) ) perform_cv_validations( algorithm_params, OpenTox::SubTask.create(task, 33, 100) ) end @@ -324,7 +344,7 @@ module Validation cvs.each do |cv| next if AA_SERVER and !OpenTox::Authorization.authorized?(cv.crossvalidation_uri,"GET",self.subjectid) tmp_val = [] - Validation.find( :crossvalidation_id => cv.id ).each do |v| + Validation.find( :crossvalidation_id => cv.id, :validation_type => "crossvalidation" ).each do |v| break unless v.prediction_feature == prediction_feature and OpenTox::Dataset.exist?(v.training_dataset_uri,self.subjectid) and @@ -353,7 +373,7 @@ module Validation # stores uris in validation objects def create_new_cv_datasets( prediction_feature, task = nil ) LOGGER.debug "creating datasets for crossvalidation" - orig_dataset = OpenTox::Dataset.find(self.dataset_uri,self.subjectid) + orig_dataset = Lib::DatasetCache.find(self.dataset_uri,self.subjectid) raise OpenTox::NotFoundError.new "Dataset not found: "+self.dataset_uri.to_s unless orig_dataset shuffled_compounds = orig_dataset.compounds.shuffle( self.random_seed ) @@ -465,7 +485,7 @@ module Validation random_seed=1 unless random_seed - orig_dataset = OpenTox::Dataset.find orig_dataset_uri,subjectid + orig_dataset = Lib::DatasetCache.find orig_dataset_uri,subjectid orig_dataset.load_all raise OpenTox::NotFoundError.new "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset if prediction_feature @@ -530,7 +550,7 @@ module Validation task.progress(100) if task if ENV['RACK_ENV'] =~ /test|debug/ - training_dataset = OpenTox::Dataset.find result[:training_dataset_uri],subjectid + training_dataset = Lib::DatasetCache.find result[:training_dataset_uri],subjectid raise OpenTox::NotFoundError.new "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless training_dataset training_dataset.load_all value_count = 0 @@ -539,7 +559,7 @@ module Validation end raise "training compounds error" unless value_count==training_compounds.size raise OpenTox::NotFoundError.new "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless - OpenTox::Dataset.find result[:test_dataset_uri], subjectid + Lib::DatasetCache.find result[:test_dataset_uri], subjectid end LOGGER.debug "bootstrapping done, training dataset: '"+result[:training_dataset_uri].to_s+"', test dataset: '"+result[:test_dataset_uri].to_s+"'" @@ -554,7 +574,7 @@ module Validation random_seed=1 unless random_seed random_seed = random_seed.to_i - orig_dataset = OpenTox::Dataset.find orig_dataset_uri, subjectid + orig_dataset = Lib::DatasetCache.find orig_dataset_uri, subjectid orig_dataset.load_all subjectid raise OpenTox::NotFoundError.new "Dataset not found: "+orig_dataset_uri.to_s unless orig_dataset raise OpenTox::NotFoundError.new "Split ratio invalid: "+split_ratio.to_s unless split_ratio and split_ratio=split_ratio.to_f @@ -597,7 +617,7 @@ module Validation subjectid ).uri task.progress(66) if task -# d = OpenTox::Dataset.find(result[:training_dataset_uri]) +# d = Lib::DatasetCache.find(result[:training_dataset_uri]) # d.data_entries.values.each do |v| # puts v.inspect # puts v.values[0].to_s+" "+v.values[0].class.to_s @@ -617,8 +637,8 @@ module Validation if ENV['RACK_ENV'] =~ /test|debug/ raise OpenTox::NotFoundError.new "Training dataset not found: '"+result[:training_dataset_uri].to_s+"'" unless - OpenTox::Dataset.find(result[:training_dataset_uri],subjectid) - test_data = OpenTox::Dataset.find result[:test_dataset_uri],subjectid + Lib::DatasetCache.find(result[:training_dataset_uri],subjectid) + test_data = Lib::DatasetCache.find result[:test_dataset_uri],subjectid raise OpenTox::NotFoundError.new "Test dataset not found: '"+result[:test_dataset_uri].to_s+"'" unless test_data test_data.load_compounds subjectid raise "Test dataset num coumpounds != "+(compounds.size-split-1).to_s+", instead: "+ -- cgit v1.2.3 From ac6a536d12697a98db6847c5229c67372cbbd1e7 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 18 May 2011 15:35:52 +0200 Subject: new feature: algorithm comparison report --- lib/merge.rb | 2 +- lib/predictions.rb | 19 ++- report/plot_factory.rb | 34 +++-- report/report_application.rb | 3 +- report/report_content.rb | 61 ++++----- report/report_factory.rb | 102 +++++++------- report/report_persistance.rb | 1 + report/report_service.rb | 7 +- report/statistical_test.rb | 2 +- report/validation_access.rb | 12 +- report/validation_data.rb | 126 ++++++++--------- report/xml_report.rb | 16 ++- test/test_examples.rb | 210 ++++++++++++++++++++++++++++- validation/validation_service.rb | 6 +- validation/validation_test.rb | 283 ++++++++++++++++++++++++++------------- 15 files changed, 599 insertions(+), 285 deletions(-) diff --git a/lib/merge.rb b/lib/merge.rb index ecbe133..f30a3c1 100644 --- a/lib/merge.rb +++ b/lib/merge.rb @@ -126,7 +126,7 @@ module Lib if value1==nil && value2==nil value = nil elsif value1.to_s != value2.to_s - value = value1.to_s + "/" + value2.to_s + value = value1.to_s + ";" + value2.to_s else value = value2.to_s end diff --git a/lib/predictions.rb b/lib/predictions.rb index 420790e..f97b764 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -170,8 +170,8 @@ module Lib def percent_correct raise "no classification" unless @feature_type=="classification" - return 0 if @num_with_actual_value==0 - return 100 * @num_correct / (@num_with_actual_value - @num_unpredicted).to_f + pct = 100 * @num_correct / (@num_with_actual_value - @num_unpredicted).to_f + pct.nan? ? 0 : pct end def percent_incorrect @@ -181,7 +181,8 @@ module Lib end def accuracy - return percent_correct / 100.0 + acc = percent_correct / 100.0 + acc.nan? ? 0 : acc end def weighted_accuracy @@ -250,6 +251,7 @@ module Lib return res end + # does only take the instances that are classified as into account def area_under_roc(class_index=nil) return prediction_feature_value_map( lambda{ |i| area_under_roc(i) } ) if class_index==nil @@ -427,8 +429,13 @@ module Lib return incorrect end + # Note: + # * (un-weighted) area under roc is computed with all __predicted__ isntances for a certain class + # * weighted weights each auc with the number of __acutal__ instances + # its like that, because its like that in weka def weighted_area_under_roc - return weighted_measure( :area_under_roc ) + w_auc = weighted_measure( :area_under_roc ) + w_auc.nan? ? 0 : w_auc end def weighted_f_measure @@ -436,6 +443,7 @@ module Lib end private + # the is weighted with the number of instances for each actual class value def weighted_measure( measure ) sum_instances = 0 @@ -478,9 +486,10 @@ module Lib def sample_correlation_coefficient # formula see http://en.wikipedia.org/wiki/Correlation_and_dependence#Pearson.27s_product-moment_coefficient - return ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) / + scc = ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) / ( Math.sqrt( [0, @num_predicted * @sum_squares_actual - @sum_actual**2].max ) * Math.sqrt( [0, @num_predicted * @sum_squares_predicted - @sum_predicted**2].max ) ) + ( scc.infinite? || scc.nan? ) ? 0 : scc end def total_sum_of_squares diff --git a/report/plot_factory.rb b/report/plot_factory.rb index b7c920a..74c89f5 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -167,25 +167,35 @@ module Reports validation_set.validations.each do |v| values = [] value_attributes.each do |a| - validation_set.get_accept_values_for_attr(a).each do |class_value| - value = v.send(a) - if value.is_a?(Hash) - if class_value==nil - avg_value = 0 - value.values.each{ |val| avg_value+=val } - value = avg_value/value.values.size.to_f - else - raise "bar plot value is hash, but no entry for class-value ("+class_value.to_s+"); value for "+a.to_s+" -> "+value.inspect unless value.key?(class_value) - value = value[class_value] + + accept = validation_set.get_accept_values_for_attr(a) + if accept and accept.size>0 + accept.each do |class_value| + value = v.send(a) + if value.is_a?(Hash) + if class_value==nil + avg_value = 0 + value.values.each{ |val| avg_value+=val } + value = avg_value/value.values.size.to_f + else + raise "bar plot value is hash, but no entry for class-value ("+class_value.to_s+"); value for "+a.to_s+" -> "+value.inspect unless value.key?(class_value) + value = value[class_value] + end end + raise "value is nil\nattribute: "+a.to_s+"\nvalidation: "+v.inspect if value==nil + values.push(value) + labels.push(a.to_s.gsub("_","-") + ( class_value==nil ? "" : "("+class_value.to_s+")" )) end - raise "value is nil\nattribute: "+a.to_s+"\nvalidation: "+v.inspect if value==nil + else + value = v.send(a) values.push(value) - labels.push(a.to_s.gsub("_","-") + ( class_value==nil ? "" : "("+class_value.to_s+")" )) + labels.push(a.to_s.gsub("_","-")) end + end titles << v.send(title_attribute).to_s + raise "no title for '"+title_attribute.to_s+"' in validation: "+v.to_yaml if titles[-1].to_s.size==0 data << values end diff --git a/report/report_application.rb b/report/report_application.rb index 258daa7..1637ead 100755 --- a/report/report_application.rb +++ b/report/report_application.rb @@ -114,7 +114,8 @@ end post '/report/:type' do task = OpenTox::Task.create("Create report",url_for("/report/"+params[:type], :full)) do |task| #,params perform do |rs| - rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil,@subjectid,task) + rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil, + params[:identifier]?params[:identifier].split(/\n|,/):nil,@subjectid,task) end end return_task(task) diff --git a/report/report_content.rb b/report/report_content.rb index cc4c13c..ac64bab 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -22,28 +22,32 @@ class Reports::ReportContent @current_section = @xml_report.get_root_element end - def add_paired_ttest_table( validation_set, + def add_paired_ttest_tables( validation_set, group_attribute, - test_attribute, + test_attributes, section_title = "Paired t-test", section_text = nil) - - level = 0.90 - test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, - group_attribute, test_attribute, "paired_ttest", level ) - #puts test_matrix.inspect - titles = test_matrix[:titles] - matrix = test_matrix[:matrix] - table = [] - #puts titles.inspect - table << [""] + titles - titles.size.times do |i| - table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") } - end - + section_test = @xml_report.add_section(@current_section, section_title) @xml_report.add_paragraph(section_test, section_text) if section_text - @xml_report.add_table(section_test, test_attribute.to_s+", significance-level: "+level.to_s, table, true, true) + + test_attributes.each do |test_attribute| + level = 0.90 + test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, + group_attribute, test_attribute, "paired_ttest", level ) + #puts test_matrix.inspect + titles = test_matrix[:titles] + matrix = test_matrix[:matrix] + table = [] + #puts titles.inspect + table << [""] + titles + titles.size.times do |i| + table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") } + end + + @xml_report.add_table(section_test, test_attribute.to_s+", significance-level: "+level.to_s+", num results: "+ + test_matrix[:num_results].to_s, table, true, true) + end end def add_predictions( validation_set, @@ -98,32 +102,13 @@ class Reports::ReportContent validation_attributes, table_title, section_title="Results", - section_text=nil, - #rem_equal_vals_attr=[], - search_for_existing_report_type=nil) + section_text=nil) + #rem_equal_vals_attr=[]) section_table = @xml_report.add_section(@current_section, section_title) @xml_report.add_paragraph(section_table, section_text) if section_text vals = validation_set.to_array(validation_attributes, true) vals = vals.collect{|a| a.collect{|v| v.to_s }} - - if (search_for_existing_report_type) - vals.size.times do |i| - #puts i - if (i==0) - vals[i] = [ "Reports" ] + vals[i] - puts vals[i].inspect - else - if search_for_existing_report_type=="validation" - vals[i] = [ validation_set.validations[i-1].validation_report_uri() ] + vals[i] - elsif search_for_existing_report_type=="crossvalidation" - vals[i] = [ validation_set.validations[i-1].cv_report_uri() ] + vals[i] - else - raise "illegal report type: "+search_for_existing_report_type.to_s - end - end - end - end #PENDING transpose values if there more than 4 columns, and there are more than columns than rows transpose = vals[0].size>4 && vals[0].size>vals.size @xml_report.add_table(section_table, table_title, vals, !transpose, transpose, transpose) diff --git a/report/report_factory.rb b/report/report_factory.rb index 08d9418..7e74cb4 100755 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -100,33 +100,37 @@ module Reports::ReportFactory raise OpenTox::BadRequestError.new("validations must have unique feature type, i.e. must be either all regression, "+ +"or all classification validations") unless validation_set.unique_feature_type pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) ) + validation_set.validations.sort! do |x,y| + x.crossvalidation_fold.to_f <=> y.crossvalidation_fold.to_f + end + cv_set = validation_set.replace_with_cv_stats + raise unless cv_set.size==1 - merged = validation_set.merge([:crossvalidation_id]) - raise unless merged.size==1 - - #puts merged.get_values(:percent_correct_variance, false).inspect + #puts cv_set.get_values(:percent_correct_variance, false).inspect report = Reports::ReportContent.new("Crossvalidation report") + res_titel = "Crossvalidation Results" + res_text = "These performance statistics have been derieved by accumulating all predictions on the various fold (i.e. these numbers are NOT averaged results over all crossvalidation folds)." case validation_set.unique_feature_type when "classification" - report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold],"Mean Results","Mean Results") - report.add_confusion_matrix(merged.validations[0]) + report.add_result(cv_set, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold], res_titel, res_titel, res_text) + report.add_confusion_matrix(cv_set.validations[0]) report.add_section("Plots") report.add_roc_plot(validation_set) report.add_roc_plot(validation_set, :crossvalidation_fold) report.add_confidence_plot(validation_set) report.add_confidence_plot(validation_set, :crossvalidation_fold) report.end_section - report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_CLASS-[:num_folds], - "Results","Results",nil,"validation") + report.add_result(validation_set, [:validation_uri, :validation_report_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:num_folds, :dataset_uri, :algorithm_uri], + "Results","Results") when "regression" - report.add_result(merged, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold],"Mean Results","Mean Results") + report.add_result(cv_set, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold],res_titel, res_titel, res_text) report.add_section("Plots") report.add_regression_plot(validation_set, :crossvalidation_fold) report.add_confidence_plot(validation_set) report.add_confidence_plot(validation_set, :crossvalidation_fold) report.end_section - report.add_result(validation_set, VAL_ATTR_CV+VAL_ATTR_REGR-[:num_folds], "Results","Results") + report.add_result(validation_set, [:validation_uri, :validation_report_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:num_folds, :dataset_uri, :algorithm_uri], "Results","Results") end task.progress(90) if task @@ -142,8 +146,8 @@ module Reports::ReportFactory raise OpenTox::BadRequestError.new("num validations is not >1") unless validation_set.size>1 raise OpenTox::BadRequestError.new("validations must have unique feature type, i.e. must be either all regression, "+ +"or all classification validations") unless validation_set.unique_feature_type - raise OpenTox::BadRequestError.new("number of different algorithms <2: "+ - validation_set.get_values(:algorithm_uri).inspect) if validation_set.num_different_values(:algorithm_uri)<2 + raise OpenTox::BadRequestError.new("number of different identifiers <2: "+ + validation_set.get_values(:identifier).inspect) if validation_set.num_different_values(:identifier)<2 if validation_set.has_nil_values?(:crossvalidation_id) raise OpenTox::BadRequestError.new("algorithm comparison for non crossvalidation not yet implemented") @@ -160,73 +164,63 @@ module Reports::ReportFactory # groups results into sets with equal dataset if (validation_set.num_different_values(:dataset_uri)>1) + LOGGER.debug "compare report -- num different datasets: "+validation_set.num_different_values(:dataset_uri).to_s dataset_grouping = Reports::Util.group(validation_set.validations, [:dataset_uri]) # check if equal values in each group exist - Reports::Util.check_group_matching(dataset_grouping, [:algorithm_uri, :crossvalidation_fold, :num_folds, :stratified, :random_seed]) + Reports::Util.check_group_matching(dataset_grouping, [:crossvalidation_fold, :num_folds, :stratified, :random_seed]) else dataset_grouping = [ validation_set.validations ] end - # we only checked that equal validations exist in each dataset group, now check for each algorithm + # we only checked that equal validations exist in each dataset group, now check for each identifier dataset_grouping.each do |validations| - algorithm_grouping = Reports::Util.group(validations, [:algorithm_uri]) + algorithm_grouping = Reports::Util.group(validations, [:identifier]) Reports::Util.check_group_matching(algorithm_grouping, [:crossvalidation_fold, :num_folds, :stratified, :random_seed]) end pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) ) - report = Reports::ReportContent.new("Algorithm comparison report - Many datasets") + report = Reports::ReportContent.new("Algorithm comparison report") if (validation_set.num_different_values(:dataset_uri)>1) all_merged = validation_set.merge([:algorithm_uri, :dataset_uri, :crossvalidation_id, :crossvalidation_uri]) report.add_ranking_plots(all_merged, :algorithm_uri, :dataset_uri, [:percent_correct, :weighted_area_under_roc, :true_positive_rate, :true_negative_rate] ) report.add_result_overview(all_merged, :algorithm_uri, :dataset_uri, [:percent_correct, :weighted_area_under_roc, :true_positive_rate, :true_negative_rate]) - end - + + result_attributes = [:identifier,:crossvalidation_uri,:crossvalidation_report_uri]+VAL_ATTR_CV-[:crossvalidation_fold,:num_folds,:dataset_uri] case validation_set.unique_feature_type when "classification" - attributes = VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold] - attributes = ([ :dataset_uri ] + attributes).uniq - - dataset_grouping.each do |validations| - - set = Reports::ValidationSet.create(validations) - - dataset = validations[0].dataset_uri - merged = set.merge([:algorithm_uri, :dataset_uri, :crossvalidation_id, :crossvalidation_uri]) - merged.sort(:dataset_uri) - - report.add_section("Dataset: "+dataset) - report.add_result(merged,attributes, - "Mean Results","Mean Results",nil,"crossvalidation") - report.add_paired_ttest_table(set, :algorithm_uri, :percent_correct) - - report.add_bar_plot(merged, :algorithm_uri, VAL_ATTR_BAR_PLOT_CLASS) - report.add_roc_plot(set, :algorithm_uri) - report.end_section - end - - when "regression" + result_attributes += VAL_ATTR_CLASS + ttest_attributes = [:percent_correct, :weighted_area_under_roc] + bar_plot_attributes = VAL_ATTR_BAR_PLOT_CLASS + else + result_attributes += VAL_ATTR_REGR + ttest_attributes = [:r_square, :root_mean_squared_error] + bar_plot_attributes = VAL_ATTR_BAR_PLOT_REGR + end - attributes = VAL_ATTR_CV+VAL_ATTR_REGR-[:crossvalidation_fold] - attributes = ([ :dataset_uri ] + attributes).uniq + dataset_grouping.each do |validations| + + set = Reports::ValidationSet.create(validations) - dataset_grouping.each do |validations| + dataset = validations[0].dataset_uri + merged = set.merge([:identifier, :dataset_uri]) #, :crossvalidation_id, :crossvalidation_uri]) + merged.sort(:identifier) - set = Reports::ValidationSet.create(validations) - - dataset = validations[0].dataset_uri - merged = set.merge([:algorithm_uri, :dataset_uri, :crossvalidation_id, :crossvalidation_uri]) - merged.sort(:dataset_uri) - - report.add_section("Dataset: "+dataset) - report.add_result(merged,attributes, - "Mean Results","Mean Results",nil,"crossvalidation") - report.add_paired_ttest_table(set, :algorithm_uri, :r_square) - report.end_section + merged.validations.each do |v| + v.crossvalidation_uri = v.crossvalidation_uri.split(";").uniq.join(" ") + v.crossvalidation_report_uri = v.crossvalidation_report_uri.split(";").uniq.join(" ") if v.crossvalidation_report_uri end + report.add_section("Dataset: "+dataset) + res_titel = "Average Results on Folds" + res_text = "These performance statistics have been derieved by computing the mean of the statistics on each crossvalidation fold." + report.add_result(merged,result_attributes,res_titel,res_titel,res_text) + # pending: regression stats have different scales!!! + report.add_bar_plot(merged, :identifier, bar_plot_attributes) if validation_set.unique_feature_type=="classification" + report.add_paired_ttest_tables(set, :identifier, ttest_attributes) + report.end_section end task.progress(100) if task report diff --git a/report/report_persistance.rb b/report/report_persistance.rb index c85ad68..78ae47b 100755 --- a/report/report_persistance.rb +++ b/report/report_persistance.rb @@ -250,6 +250,7 @@ module Reports end def list_reports(type, filter_params={}) + filter_params[:report_type] = type LOGGER.debug "find reports for params: "+filter_params.inspect reports = Lib::OhmUtil.find( ReportData, filter_params ) reports.collect{ |r| r.id } diff --git a/report/report_service.rb b/report/report_service.rb index 722c3d6..3e23889 100644 --- a/report/report_service.rb +++ b/report/report_service.rb @@ -60,7 +60,7 @@ module Reports # call-seq: # create_report(type, validation_uris) => string # - def create_report(type, validation_uris, subjectid=nil, task=nil) + def create_report(type, validation_uris, identifier=nil, subjectid=nil, task=nil) LOGGER.info "create report of type '"+type.to_s+"'" check_report_type(type) @@ -68,7 +68,10 @@ module Reports # step1: load validations raise OpenTox::BadRequestError.new("validation_uris missing") unless validation_uris LOGGER.debug "validation_uri(s): '"+validation_uris.inspect+"'" - validation_set = Reports::ValidationSet.new(validation_uris, subjectid) + LOGGER.debug "identifier: '"+identifier.inspect+"'" + raise "illegal num identifiers: "+identifier.size.to_s+" should be equal to num validation-uris ("+validation_uris.size.to_s+")" if + identifier and identifier.size!=validation_uris.size + validation_set = Reports::ValidationSet.new(validation_uris, identifier, subjectid) raise OpenTox::BadRequestError.new("cannot get validations from validation_uris '"+validation_uris.inspect+"'") unless validation_set and validation_set.size > 0 LOGGER.debug "loaded "+validation_set.size.to_s+" validation/s" task.progress(10) if task diff --git a/report/statistical_test.rb b/report/statistical_test.rb index c37827e..1e586e2 100644 --- a/report/statistical_test.rb +++ b/report/statistical_test.rb @@ -58,7 +58,7 @@ module Reports end end end - {:titles => titles, :matrix => matrix} + {:titles => titles, :matrix => matrix, :num_results => grouped_validations[0].size} end def self.paired_ttest( validations1, validations2, attribute, significance_level=0.95 ) diff --git a/report/validation_access.rb b/report/validation_access.rb index d0c3a1d..ffb7461 100755 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -7,8 +7,9 @@ require "lib/validation_db.rb" # class Reports::ValidationDB - def resolve_cv_uris(validation_uris, subjectid=nil) - res = [] + def resolve_cv_uris(validation_uris, identifier=nil, subjectid=nil) + res = {} + count = 0 validation_uris.each do |u| if u.to_s =~ /.*\/crossvalidation\/[0-9]+/ cv_id = u.split("/")[-1].to_i @@ -25,10 +26,13 @@ class Reports::ValidationDB raise OpenTox::NotFoundError.new "crossvalidation with id "+cv_id.to_s+" not found" unless cv raise OpenTox::BadRequestError.new("crossvalidation with id '"+cv_id.to_s+"' not finished") unless cv.finished #res += Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s} - res += Validation::Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|v| v.validation_uri.to_s } + Validation::Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).each do |v| + res[v.validation_uri.to_s] = identifier ? identifier[count] : nil + end else - res += [u.to_s] + res[u.to_s] = identifier ? identifier[count] : nil end + count += 1 end res end diff --git a/report/validation_data.rb b/report/validation_data.rb index 11fa737..0c15dd2 100755 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -1,6 +1,7 @@ # the variance is computed when merging results for these attributes -VAL_ATTR_VARIANCE = [ :area_under_roc, :percent_correct, :root_mean_squared_error, :mean_absolute_error, :r_square, :accuracy ] +VAL_ATTR_VARIANCE = [ :area_under_roc, :percent_correct, :root_mean_squared_error, :mean_absolute_error, + :r_square, :accuracy, :weighted_area_under_roc, :weighted_accuracy ] VAL_ATTR_RANKING = [ :area_under_roc, :percent_correct, :true_positive_rate, :true_negative_rate, :weighted_area_under_roc ] #:accuracy ] ATTR_NICE_NAME = {} @@ -51,21 +52,31 @@ end module Reports + @@validation_access = ValidationDB.new + @@persistance = ReportService.persistance + + def self.persistance + @@persistance + end + + def self.validation_access + @@validation_access + end + + # for overwriting validation source (other than using webservices) + def self.reset_validation_access(validation_access) + @@validation_access = validation_access + end + + # = ReportValidation # # contains all values of a validation object # class ReportValidation - @@validation_access = ValidationDB.new - - # for overwriting validation source (other than using webservices) - def self.reset_validation_access(validation_access) - @@validation_access = validation_access - end - - def self.resolve_cv_uris(validation_uris, subjectid) - @@validation_access.resolve_cv_uris(validation_uris, subjectid) + def self.resolve_cv_uris(validation_uris, identifier, subjectid) + Reports.validation_access.resolve_cv_uris(validation_uris, identifier, subjectid) end # create member variables for all validation properties @@ -75,16 +86,17 @@ module Reports @@validation_attributes.each{ |a| attr_accessor a } attr_reader :predictions + attr_accessor :identifier, :validation_report_uri, :crossvalidation_report_uri def initialize(uri = nil, subjectid = nil) - @@validation_access.init_validation(self, uri, subjectid) if uri + Reports.validation_access.init_validation(self, uri, subjectid) if uri @subjectid = subjectid #raise "subjectid is nil" unless subjectid end def self.from_cv_statistics( cv_uri, subjectid = nil ) v = ReportValidation.new(nil, subjectid) - @@validation_access.init_validation_from_cv_statistics(v, cv_uri, subjectid) + Reports.validation_access.init_validation_from_cv_statistics(v, cv_uri, subjectid) v end @@ -103,7 +115,7 @@ module Reports task.progress(100) if task nil else - @predictions = @@validation_access.get_predictions( self, @subjectid, task ) + @predictions = Reports.validation_access.get_predictions( self, @subjectid, task ) end end end @@ -111,7 +123,7 @@ module Reports # returns the predictions feature values (i.e. the domain of the class attribute) # def get_accept_values() - @accept_values = @@validation_access.get_accept_values(self, @subjectid) unless @accept_values + @accept_values = Reports.validation_access.get_accept_values(self, @subjectid) unless @accept_values @accept_values end @@ -119,36 +131,21 @@ module Reports # def feature_type return @feature_type if @feature_type!=nil - @feature_type = @@validation_access.feature_type(self, @subjectid) + @feature_type = Reports.validation_access.feature_type(self, @subjectid) end def predicted_variable return @predicted_variable if @predicted_variable!=nil - @predicted_variable = @@validation_access.predicted_variable(self, @subjectid) + @predicted_variable = Reports.validation_access.predicted_variable(self, @subjectid) end # loads all crossvalidation attributes, of the corresponding cv into this object def load_cv_attributes raise "crossvalidation-id not set" unless @crossvalidation_id - @@validation_access.init_cv(self) - end - - @@persistance = ReportService.persistance - - def validation_report_uri - #puts "searching for validation report: "+self.validation_uri.to_s - return @validation_report_uri if @validation_report_uri!=nil - ids = @@persistance.list_reports("validation",{:validation_uris=>validation_uri }) - @validation_report_uri = ReportService.instance.get_uri("validation",ids[-1]) if ids and ids.size>0 - end - - def cv_report_uri - #puts "searching for cv report: "+self.crossvalidation_uri.to_s - return @cv_report_uri if @cv_report_uri!=nil - raise "no cv uri "+to_yaml unless self.crossvalidation_uri - ids = @@persistance.list_reports("crossvalidation",{:crossvalidation=>self.crossvalidation_uri.to_s }) - #puts "-> "+ids.inspect - @cv_report_uri = ReportService.instance.get_uri("crossvalidation",ids[-1]) if ids and ids.size>0 + Reports.validation_access.init_cv(self) + # load cv report + ids = Reports.persistance.list_reports("crossvalidation",{:crossvalidation=>self.crossvalidation_uri.to_s }) + @crossvalidation_report_uri = ReportService.instance.get_uri("crossvalidation",ids[-1]) if ids and ids.size>0 end def clone_validation @@ -164,13 +161,20 @@ module Reports # class ValidationSet - def initialize(validation_uris=nil, subjectid=nil) + def initialize(validation_uris=nil, identifier=nil, subjectid=nil) @unique_values = {} - validation_uris = ReportValidation.resolve_cv_uris(validation_uris, subjectid) if validation_uris - @validations = Array.new - validation_uris.each{|u| @validations.push(ReportValidation.new(u, subjectid))} if validation_uris + @validations = [] + if validation_uris + validation_uri_and_ids = ReportValidation.resolve_cv_uris(validation_uris, identifier, subjectid) + validation_uri_and_ids.each do |u,id| + v = ReportValidation.new(u, subjectid) + v.identifier = id if id + ids = Reports.persistance.list_reports("validation",{:validation_uris=>v.validation_uri }) + v.validation_report_uri = ReportService.instance.get_uri("validation",ids[-1]) if ids and ids.size>0 + @validations << v + end + end end - def self.create(validations) set = ValidationSet.new @@ -399,6 +403,17 @@ module Reports return array end + def replace_with_cv_stats + new_set = ValidationSet.new + grouping = Util.group(@validations, [:crossvalidation_id]) + grouping.each do |g| + v = ReportValidation.from_cv_statistics(g[0].crossvalidation_uri) + v.identifier = g.collect{|vv| vv.identifier}.uniq.join(";") + new_set.validations << v + end + return new_set + end + # creates a new validaiton set, that contains merged validations # all validation with equal values for __equal_attributes__ are summed up in one validation, i.e. merged # @@ -416,31 +431,16 @@ module Reports grouping = Util.group(@validations, equal_attributes) #puts "groups "+grouping.size.to_s - if ( equal_attributes.include?(:crossvalidation_id) ) - # do not merge, use crossvalidation statistics - raise "statistics vs merging problem" if equal_attributes.size!=1 - grouping.each do |g| - new_set.validations << ReportValidation.from_cv_statistics(g[0].crossvalidation_uri) - end - else - #merge - Lib::MergeObjects.register_merge_attributes( ReportValidation, - Validation::VAL_MERGE_AVG,Validation::VAL_MERGE_SUM,Validation::VAL_MERGE_GENERAL) unless - Lib::MergeObjects.merge_attributes_registered?(ReportValidation) - grouping.each do |g| - new_set.validations << g[0].clone_validation - w = 1 - g[1..-1].each do |v| - new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v,w,1) - w+=1 - end + #merge + Lib::MergeObjects.register_merge_attributes( ReportValidation, + Validation::VAL_MERGE_AVG+Validation::VAL_MERGE_SUM,[],Validation::VAL_MERGE_GENERAL+[:identifier, :validation_report_uri, :crossvalidation_report_uri]) unless + Lib::MergeObjects.merge_attributes_registered?(ReportValidation) + grouping.each do |g| + new_set.validations << g[0].clone_validation + g[1..-1].each do |v| + new_set.validations[-1] = Lib::MergeObjects.merge_objects(new_set.validations[-1],v) end end - - new_set.validations.each do |v| - raise "not a validation "+v.class.to_s+" "+v.to_s unless v.is_a?(Reports::ReportValidation) - end - return new_set end diff --git a/report/xml_report.rb b/report/xml_report.rb index 4fbfae3..7467c47 100755 --- a/report/xml_report.rb +++ b/report/xml_report.rb @@ -187,7 +187,13 @@ module Reports if auto_link_urls && v.to_s =~ /depict/ || v.to_s =~ /image\/png$/ #PENDING add_image(entry, v.to_s) elsif auto_link_urls && v.to_s =~ /^http(s?):\/\// - add_url(entry, v.to_s, v.to_s) + #add_url(entry, v.to_s, v.to_s) + v.to_s.split(" ").each do |vv| + add_url(entry, vv.to_s, vv.to_s) + space = Element.new("para") + space.text = " " + entry << space + end else entry.text = v.to_s end @@ -221,11 +227,15 @@ module Reports return list end - def add_url (element, url, description=url ) - + def url_element( url, description=url ) ulink = Element.new("ulink") ulink.add_attributes({"url" => url}) ulink.text = description + ulink + end + + def add_url (element, url, description=url ) + ulink = url_element(url, description) element << ulink return ulink end diff --git a/test/test_examples.rb b/test/test_examples.rb index 49d7838..eb0543f 100755 --- a/test/test_examples.rb +++ b/test/test_examples.rb @@ -74,6 +74,14 @@ module ValidationExamples end end + class LazarLastEPAFHMSplit < LazarEPAFHMSplit + def initialize + super + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/last") + end + end + + class MajorityEPAFHMSplit < EPAFHMSplit def initialize @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") @@ -81,11 +89,18 @@ module ValidationExamples end end + class MajorityRandomEPAFHMSplit < MajorityEPAFHMSplit + def initialize + @algorithm_params = "random=true" + super + end + end + ######################################################################################################## class EPAFHMCrossvalidation < CrossValidation def initialize - @dataset_file = File.new("data/EPAFHM.mini.csv","r") + @dataset_file = File.new("data/EPAFHM.med.csv","r") #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" @num_folds = 10 end @@ -93,11 +108,20 @@ module ValidationExamples class MajorityEPAFHMCrossvalidation < EPAFHMCrossvalidation def initialize + #@dataset_uri = "http://local-ot/dataset/2366" + #@prediction_feature = "http://local-ot/dataset/2366/feature/LC50_mmol" @algorithm_uri = File.join(CONFIG[:services]["opentox-majority"],"/regr/algorithm") super end end + class MajorityRandomEPAFHMCrossvalidation < MajorityEPAFHMCrossvalidation + def initialize + @algorithm_params = "random=true" + super + end + end + class LazarEPAFHMCrossvalidation < EPAFHMCrossvalidation def initialize @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") @@ -125,6 +149,14 @@ module ValidationExamples super end end + + class LazarLastHamsterSplit < LazarHamsterSplit + def initialize + super + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/last") + end + end + class MajorityHamsterSplit < HamsterSplit def initialize @@ -133,6 +165,13 @@ module ValidationExamples end end + class MajorityRandomHamsterSplit < MajorityHamsterSplit + def initialize + @algorithm_params = "random=true" + super + end + end + ######################################################################################################## class HamsterBootstrapping < BootstrappingValidation @@ -212,6 +251,13 @@ module ValidationExamples super end end + + class MajorityRandomHamsterCrossvalidation < MajorityHamsterCrossvalidation + def initialize + @algorithm_params = "random=true" + super + end + end class LazarHamsterCrossvalidation < HamsterCrossvalidation def initialize @@ -221,6 +267,45 @@ module ValidationExamples end end + class LazarLastHamsterCrossvalidation < LazarHamsterCrossvalidation + def initialize + super + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/last") + end + end + + ######################################################################################################## + + class LazarHamsterMiniCrossvalidation < CrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + @dataset_file = File.new("data/hamster_carcinogenicity.mini.csv","r") + @num_folds = 2 + end + end + + class ISSCANStratifiedCrossvalidation < CrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + @dataset_file = File.new("data/ISSCAN_v3a_canc-red.csv","r") + @stratified = true + @num_folds = 10 + end + end + + class ISSCAN2StratifiedCrossvalidation < CrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + @dataset_file = File.new("data/ISSCAN_v3a_sal.csv","r") + @stratified = true + @num_folds = 10 + end + end + + ######################################################################################################## class ISTHamsterCrossvalidation < CrossValidation @@ -368,6 +453,74 @@ module ValidationExamples end end + class NtuaModel2 < ModelValidation + def initialize + @model_uri = "http://opentox.ntua.gr:8080/model/11093fbc-3b8b-41e2-bfe3-d83f5f529efc" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/54" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/579820" + end + end + + class NtuaModel3 < ModelValidation + def initialize + @model_uri = "http://opentox.ntua.gr:8080/model/bbab3714-e90b-4990-bef9-8e7d3a30eece" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + #@prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/579820" + end + end + + ######################################################################################################## + + class NtuaTrainingTest < TrainingTestValidation + def initialize + @algorithm_uri = "http://opentox.ntua.gr:8080/algorithm/mlr" + @training_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + end + end + + class NtuaTrainingTestSplit < SplitTestValidation + def initialize + @algorithm_uri = "http://opentox.ntua.gr:8080/algorithm/mlr" + @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + end + end + + class NtuaCrossvalidation < CrossValidation + def initialize + @algorithm_uri = "http://opentox.ntua.gr:8080/algorithm/mlr" + @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + end + end + + class AmbitVsNtuaTrainingTest < TrainingTestValidation + def initialize + @algorithm_uri = "http://apps.ideaconsult.net:8080/ambit2/algorithm/LR" + @training_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + end + end + + + class LazarVsNtuaCrossvalidation < CrossValidation + def initialize + @algorithm_uri = File.join(CONFIG[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(CONFIG[:services]["opentox-algorithm"],"fminer/bbrc") + @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + @prediction_feature="http://apps.ideaconsult.net:8080/ambit2/feature/22200" + @num_folds=3 + end + end + + +# loading prediciton via test-dataset:'http://apps.ideaconsult.net:8080/ambit2/dataset/R545', +# test-target-datset:'', prediction-dataset:'http://apps.ideaconsult.net:8080/ambit2/dataset/584389', +# prediction_feature: 'http://apps.ideaconsult.net:8080/ambit2/feature/22200' ', predicted_variable: 'http://apps.ideaconsult.net:8080/ambit2/feature/627667' :: /ot_predictions.rb:21:in `initialize' +#D, [2011-05-11T13:47:26.631628 #22952] DEBUG -- : validation :: ######################################################################################################## class TumModel < ModelValidation @@ -402,6 +555,23 @@ module ValidationExamples end end + class AmbitXYModelValidation < ModelValidation + def initialize + @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/237692" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R736156" + @prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/430905" + end + end + + class AmbitXYZModelValidation < ModelValidation + def initialize + @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/238008" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R736396" + #@prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/430905" ?? + end + end + + class AmbitTrainingTest < TrainingTestValidation def initialize @training_dataset_uri = "https://ambit.uni-plovdiv.bg:8443/ambit2/dataset/R401560" @@ -528,20 +698,36 @@ module ValidationExamples end end + ######################################################################################################## + + class TumCrossValidation < CrossValidation + def initialize + @dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/124963" + @algorithm_uri = "http://opentox:8080/OpenTox/algorithm/kNNregression" + @prediction_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/121905" + @num_folds=2 + super + end + end + ######################################################################################################## @@list = { - "1" => [ LazarHamsterSplit, MajorityHamsterSplit ], + "1" => [ LazarHamsterSplit, MajorityHamsterSplit, MajorityRandomHamsterSplit ], "1a" => [ LazarHamsterSplit ], "1b" => [ MajorityHamsterSplit ], + "1c" => [ MajorityRandomHamsterSplit ], + "1d" => [ LazarLastHamsterSplit ], "2" => [ LazarHamsterTrainingTest, MajorityHamsterTrainingTest ], "2a" => [ LazarHamsterTrainingTest ], "2b" => [ MajorityHamsterTrainingTest ], - "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation ], + "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation, MajorityRandomHamsterCrossvalidation ], "3a" => [ LazarHamsterCrossvalidation ], "3b" => [ MajorityHamsterCrossvalidation ], + "3c" => [ MajorityRandomHamsterCrossvalidation ], + "3d" => [ LazarLastHamsterCrossvalidation ], "4" => [ MajorityISTHamsterCrossvalidation, LazarISTHamsterCrossvalidation, ISTLazarISTHamsterCrossvalidation ], "4a" => [ MajorityISTHamsterCrossvalidation ], @@ -574,11 +760,17 @@ module ValidationExamples "13a" => [ LazarEPAFHMSplit ], "13b" => [ MajorityEPAFHMSplit ], + "13c" => [ MajorityRandomEPAFHMSplit ], + "13d" => [ LazarLastEPAFHMSplit ], + "14" => [ LazarEPAFHMCrossvalidation, MajorityEPAFHMCrossvalidation, MajorityRandomEPAFHMCrossvalidation ], "14a" => [ LazarEPAFHMCrossvalidation ], "14b" => [ MajorityEPAFHMCrossvalidation ], + "14c" => [ MajorityRandomEPAFHMCrossvalidation ], "15a" => [ NtuaModel ], + "15b" => [ NtuaModel2 ], + "15c" => [ NtuaModel3 ], "16" => [ LazarRepdoseSplit, MajorityRepdoseSplit ], "16a" => [ LazarRepdoseSplit ], @@ -599,7 +791,19 @@ module ValidationExamples "19g" => [ AmbitJ48TrainingTest ], "19h" => [ AmbitJ48TrainingTestSplit ], "19i" => [ AmbitAquaticModelValidation ], + "19j" => [ AmbitXYModelValidation ], + + "20a" => [ TumCrossValidation ], + + "21a" => [ LazarHamsterMiniCrossvalidation ], + "21b" => [ ISSCANStratifiedCrossvalidation ], + "21c" => [ ISSCAN2StratifiedCrossvalidation ], + "22a" => [ NtuaTrainingTest ], + "22b" => [ NtuaTrainingTestSplit ], + "22c" => [ NtuaCrossvalidation ], + "22d" => [ LazarVsNtuaCrossvalidation ], + "22e" => [ AmbitVsNtuaTrainingTest ], } diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 99d8672..520d84c 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -63,13 +63,13 @@ module Validation :percent_unpredicted => prediction.percent_unpredicted, :finished => true (VAL_PROPS_GENERAL-[:validation_uri]).each do |p| - v.send("#{p.to_s}=".to_sym, vals.collect{ |vv| vv.send(p) }.uniq.join(",")) + v.send("#{p.to_s}=".to_sym, vals.collect{ |vv| vv.send(p) }.uniq.join(";")) end v.date = crossvalidation.date v.validation_type = "crossvalidation_statistics" v.crossvalidation_id = crossvalidation.id - v.crossvalidation_fold = vals.collect{ |vv| vv.crossvalidation_fold }.uniq.join(",") - v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(",") + v.crossvalidation_fold = vals.collect{ |vv| vv.crossvalidation_fold }.uniq.join(";") + v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(";") v.save end v diff --git a/validation/validation_test.rb b/validation/validation_test.rb index efa8ad5..2c86548 100755 --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -8,7 +8,7 @@ before { require "uri" require "yaml" -ENV['RACK_ENV'] = 'test' +ENV['RACK_ENV'] = 'production' require 'application.rb' require 'test/unit' require 'rack/test' @@ -20,10 +20,10 @@ LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S " LOGGER.formatter = Logger::Formatter.new if AA_SERVER - TEST_USER = "mgtest" - TEST_PW = "mgpasswd" - #TEST_USER = "guest" - #TEST_PW = "guest" + #TEST_USER = "mgtest" + #TEST_PW = "mgpasswd" + TEST_USER = "guest" + TEST_PW = "guest" SUBJECTID = OpenTox::Authorization.authenticate(TEST_USER,TEST_PW) raise "could not log in" unless SUBJECTID puts "logged in: "+SUBJECTID.to_s @@ -60,6 +60,171 @@ class ValidationTest < Test::Unit::TestCase begin $test_case = self + #get 'crossvalidation/138/statistics' + #get 'crossvalidation/189/statistics' + #puts last_response.body +# run_test("1b") + + #get '/crossvalidation/79/predictions',nil,'HTTP_ACCEPT' => "application/x-yaml" + #puts last_response.body + + #run_test("22e") #,:validation_uri=>"http://local-ot/validation/84" ) + #run_test("21b") + #run_test("21c") + + # get '?media=text/uri-list' + + #post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/135,http://local-ot/validation/crossvalidation/134"} + #post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/174,http://local-ot/validation/crossvalidation/175"} + # 2 majority, 175 is real maj, 176 is random + +# post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/185,http://local-ot/validation/crossvalidation/193,http://local-ot/validation/crossvalidation/186,http://local-ot/validation/crossvalidation/194,http://local-ot/validation/crossvalidation/187,http://local-ot/validation/crossvalidation/195", +# :identifier=>"lazar,lazar,real_majority,real_majority,random_classification,random_classification"} +# uri = last_response.body +# rep = wait_for_task(uri) +# puts rep + +# post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/199,http://local-ot/validation/crossvalidation/204,http://local-ot/validation/crossvalidation/203", +# :identifier=>"lazar,real_majority,random_classification"} +# uri = last_response.body +# rep = wait_for_task(uri) +# puts rep + # 205 206 207 + +# post '/report/algorithm_comparison',{:validation_uris=>"http://local-ot/validation/crossvalidation/149,http://local-ot/validation/crossvalidation/210", +# :identifier=>"bbrc,last"} +# uri = last_response.body +# rep = wait_for_task(uri) +# puts rep + + #run_test("1a", {:validation_uri=>"http://local-ot/validation/466"}) +# puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + #run_test("3a",{:validation_uri=>"http://local-ot/validation/crossvalidation/149"}) + #puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + #run_test("13a", {:validation_uri=>"http://local-ot/validation/406"}) +# puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + #run_test("14a",{:validation_uri=>"http://local-ot/validation/crossvalidation/148"}) +# puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + + run_test("1a") + +# run_test("3d",{ +# :dataset_uri => "http://local-ot/dataset/2897", +# :prediction_feature => "http://local-ot/dataset/2897/feature/Hamster%20Carcinogenicity", +# :random_seed => 1 +# }) + + #run_test("14",{ + # :dataset_uri => "http://local-ot/dataset/3877", + # :prediction_feature => "http://local-ot/dataset/3877/feature/LC50_mmol", + # :random_seed => 2 + # }) + #puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + +# get "?model=http://local-ot/model/330" +# puts last_response.body +# puts "\n\n" +# get "" +# puts last_response.body + + #get "report/validation?validation=http://local-ot/validation/167" + #puts last_response.body + +# run_test("3a") #,:validation_uri=>"http://local-ot/validation/84" ) + #get "report/crossvalidation?crossvalidation=http://local-ot/validation/crossvalidation/47" + #puts last_response.body + + + rescue => ex + rep = OpenTox::ErrorReport.create(ex, "") + puts rep.to_yaml + ensure + #OpenTox::Authorization.logout(SUBJECTID) if AA_SERVER + end + end + + def app + Sinatra::Application + end + + def run_test(select=nil, overwrite={}, delete=false ) + + if AA_SERVER && SUBJECTID && delete + policies_before = OpenTox::Authorization.list_policy_uris(SUBJECTID) + end + + puts ValidationExamples.list unless select + validationExamples = ValidationExamples.select(select) + validationExamples.each do |vv| + vv.each do |v| + ex = v.new + ex.subjectid = SUBJECTID + + overwrite.each do |k,v| + ex.send(k.to_s+"=",v) + end + + unless ex.validation_uri + ex.upload_files + ex.check_requirements + ex.validate + + LOGGER.debug "validation done '"+ex.validation_uri.to_s+"'" + end + if !delete and ex.validation_uri + if SUBJECTID + puts ex.validation_uri+"?subjectid="+CGI.escape(SUBJECTID) + else + puts ex.validation_uri + end + end + + unless ex.report_uri + ex.report + end + if !delete and ex.report_uri + if SUBJECTID + puts ex.report_uri+"?subjectid="+CGI.escape(SUBJECTID) + else + puts ex.report_uri + end + end + ##ex.verify_yaml + ##ex.compare_yaml_vs_rdf + ex.delete if delete + end + end + + if AA_SERVER && SUBJECTID && delete + policies_after= OpenTox::Authorization.list_policy_uris(SUBJECTID) + diff = policies_after.size - policies_before.size + if (diff != 0) + policies_before.each do |k,v| + policies_after.delete(k) + end + LOGGER.warn diff.to_s+" policies NOT deleted:\n"+policies_after.collect{|k,v| k.to_s+" => "+v.to_s}.join("\n") + else + LOGGER.debug "all policies deleted" + end + end + end + + def prepare_examples + get '/prepare_examples' + end + + def do_test_examples # USES CURL, DO NOT FORGET TO RESTART + post '/test_examples' + end + + def do_test_examples_ortona + post '/test_examples',:examples=>"http://ortona.informatik.uni-freiburg.de/validation/examples" + end + +end + + + # prediction_feature = "https://ambit.uni-plovdiv.bg:8443/ambit2/feature/26221" # puts OpenTox::Feature.find(prediction_feature).domain.inspect # exit @@ -137,6 +302,18 @@ class ValidationTest < Test::Unit::TestCase # :regression=>"true"} # #:classification=>"true"} # puts last_response.body + +# post "/validate_datasets",{ +# :test_dataset_uri=>"http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=10", +# :prediction_dataset_uri=>"http://apps.ideaconsult.net:8080/ambit2/dataset/9?max=10", +# #:test_target_dataset_uri=>"http://local-ot/dataset/202", +# :prediction_feature=>"http://apps.ideaconsult.net:8080/ambit2/feature/21573", +# :predicted_feature=>"http://apps.ideaconsult.net:8080/ambit2/feature/21573", +# #:regression=>"true"} +# :classification=>"true"} +# puts last_response.body + + #run_test("1a") #,:validation_uri=>"http://local-ot/validation/84" ) # post "/validate_datasets",{ # :test_dataset_uri=>"http://local-ot/dataset/89", @@ -251,13 +428,17 @@ class ValidationTest < Test::Unit::TestCase #puts "" #puts last_response.body #exit + +# run_test("20a") # get "/error" # puts last_response.body #delete "/1",:subjectid=>SUBJECTID - prepare_examples() + #prepare_examples() + + #run_test("15b") #run_test("1a") #,{:validation_uri => "http://local-ot/validation/crossvalidation/1"}) @@ -275,92 +456,4 @@ class ValidationTest < Test::Unit::TestCase #prepare_examples #do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE - #do_test_examples_ortona - - rescue => ex - rep = OpenTox::ErrorReport.create(ex, "") - puts rep.to_yaml - ensure - #OpenTox::Authorization.logout(SUBJECTID) if AA_SERVER - end - end - - def app - Sinatra::Application - end - - def run_test(select=nil, overwrite={}, delete=false ) - - if AA_SERVER && SUBJECTID && delete - policies_before = OpenTox::Authorization.list_policy_uris(SUBJECTID) - end - - puts ValidationExamples.list unless select - validationExamples = ValidationExamples.select(select) - validationExamples.each do |vv| - vv.each do |v| - ex = v.new - ex.subjectid = SUBJECTID - - overwrite.each do |k,v| - ex.send(k.to_s+"=",v) - end - - unless ex.validation_uri - ex.upload_files - ex.check_requirements - ex.validate - - LOGGER.debug "validation done '"+ex.validation_uri.to_s+"'" - end - if !delete and ex.validation_uri - if SUBJECTID - puts ex.validation_uri+"?subjectid="+CGI.escape(SUBJECTID) - else - puts ex.validation_uri - end - end - - unless ex.report_uri - ex.report - end - if !delete and ex.report_uri - if SUBJECTID - puts ex.report_uri+"?subjectid="+CGI.escape(SUBJECTID) - else - puts ex.report_uri - end - end - ##ex.verify_yaml - ##ex.compare_yaml_vs_rdf - ex.delete if delete - end - end - - if AA_SERVER && SUBJECTID && delete - policies_after= OpenTox::Authorization.list_policy_uris(SUBJECTID) - diff = policies_after.size - policies_before.size - if (diff != 0) - policies_before.each do |k,v| - policies_after.delete(k) - end - LOGGER.warn diff.to_s+" policies NOT deleted:\n"+policies_after.collect{|k,v| k.to_s+" => "+v.to_s}.join("\n") - else - LOGGER.debug "all policies deleted" - end - end - end - - def prepare_examples - get '/prepare_examples' - end - - def do_test_examples # USES CURL, DO NOT FORGET TO RESTART - post '/test_examples' - end - - def do_test_examples_ortona - post '/test_examples',:examples=>"http://ortona.informatik.uni-freiburg.de/validation/examples" - end - -end + #do_test_examples_ortona \ No newline at end of file -- cgit v1.2.3 From b0ca028f1d9eaa30774f8c843c8b2ec8943247f6 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 19 May 2011 17:24:55 +0200 Subject: extended validation report plotting: * roc plot for predictions * confidence plots for class-values * aligned roc plots and confidence plots * added confidence label into roc plots --- report/plot_factory.rb | 42 +++++++++++++++++++++++----- report/report_content.rb | 71 +++++++++++++++++++++--------------------------- report/report_factory.rb | 23 +++++++++++----- report/xml_report.rb | 66 ++++++++++++++++++++++++++++++++------------ 4 files changed, 131 insertions(+), 71 deletions(-) diff --git a/report/plot_factory.rb b/report/plot_factory.rb index 74c89f5..7535eb4 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -91,7 +91,8 @@ module Reports # * the validation set is splitted into sets of validation_sets with equal attribute values # * each of theses validation sets is plotted as a roc-curve # - def self.create_roc_plot( out_file, validation_set, class_value, split_set_attribute=nil, show_single_curves=false ) + def self.create_roc_plot( out_file, validation_set, class_value, split_set_attribute=nil, + x_label="False positive rate", y_label="True Positive Rate", show_single_curves=false ) LOGGER.debug "creating roc plot for '"+validation_set.size.to_s+"' validations, out-file:"+out_file.to_s @@ -110,10 +111,16 @@ module Reports LOGGER.warn "could not create ROC plot for "+value.to_s end end - RubyPlot::plot_lines(out_file, "ROC-Plot", "False positive rate", "True Positive Rate", names, fp_rates, tp_rates ) + RubyPlot::plot_lines(out_file, "ROC-Plot", x_label, y_label, names, fp_rates, tp_rates ) else data = transform_roc_predictions(validation_set, class_value, show_single_curves) - RubyPlot::plot_lines(out_file, "ROC-Plot", "False positive rate", "True Positive Rate", data[:names], data[:fp_rate], data[:tp_rate], data[:faint] ) + labels = [] + data[:youden].each do |points| + points.each do |point,confidence| + labels << ["confidence: "+confidence.to_nice_s, point[0], point[1]] + end + end + RubyPlot::plot_lines(out_file, "ROC-Plot", x_label, y_label, data[:names], data[:fp_rate], data[:tp_rate], data[:faint], labels ) end end @@ -275,7 +282,7 @@ module Reports if (validation_set.size > 1) - names = []; fp_rate = []; tp_rate = []; faint = [] + names = []; fp_rate = []; tp_rate = []; faint = []; youden = [] sum_roc_values = { :predicted_values => [], :actual_values => [], :confidence_values => []} (0..validation_set.size-1).each do |i| @@ -299,12 +306,13 @@ module Reports names << nil # "all" fp_rate << tp_fp_rates[:fp_rate] tp_rate << tp_fp_rates[:tp_rate] + youden << tp_fp_rates[:youden] faint << false - return { :names => names, :fp_rate => fp_rate, :tp_rate => tp_rate, :faint => faint } + return { :names => names, :fp_rate => fp_rate, :tp_rate => tp_rate, :faint => faint, :youden => youden } else roc_values = validation_set.validations[0].get_predictions.get_prediction_values(class_value) tp_fp_rates = get_tp_fp_rates(roc_values) - return { :names => ["default"], :fp_rate => [tp_fp_rates[:fp_rate]], :tp_rate => [tp_fp_rates[:tp_rate]] } + return { :names => ["default"], :fp_rate => [tp_fp_rates[:fp_rate]], :tp_rate => [tp_fp_rates[:tp_rate]], :youden => [tp_fp_rates[:youden]] } end end @@ -472,13 +480,33 @@ module Reports w = w.compress_sum(c2) #puts tp_rate.inspect+"\n"+fp_rate.inspect+"\n"+w.inspect+"\n\n" + youden = [] + (0..tp_rate.size-1).each do |i| + tpr = tp_rate[i]/tp_rate[-1].to_f + fpr = fp_rate[i]/fp_rate[-1].to_f + youden << tpr + (1 - fpr) + #puts youden[-1].to_s+" ("+tpr.to_s+" "+fpr.to_s+")" + end + max = youden.max + youden_hash = {} + (0..tp_rate.size-1).each do |i| + youden_hash[i] = c2[i] if youden[i]==max + end + #puts youden.inspect+"\n"+youden_hash.inspect+"\n\n" + (0..tp_rate.size-1).each do |i| tp_rate[i] = tp_rate[-1]>0 ? tp_rate[i]/tp_rate[-1].to_f*100 : 100 fp_rate[i] = fp_rate[-1]>0 ? fp_rate[i]/fp_rate[-1].to_f*100 : 100 end #puts tp_rate.inspect+"\n"+fp_rate.inspect+"\n\n" - return {:tp_rate => tp_rate,:fp_rate => fp_rate} + youden_coordinates_hash = {} + youden_hash.each do |i,c| + youden_coordinates_hash[[fp_rate[i],tp_rate[i]]] = c + end + #puts youden_coordinates_hash.inspect+"\n\n" + + return {:tp_rate => tp_rate,:fp_rate => fp_rate,:youden => youden_coordinates_hash} end end end diff --git a/report/report_content.rb b/report/report_content.rb index ac64bab..893ac34 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -124,6 +124,11 @@ class Reports::ReportContent Reports::XMLReportUtil::create_confusion_matrix( validation.confusion_matrix ), true, true) end + # bit of a hack to algin the last two plots in the report in to one row + def align_last_two_images( title ) + @xml_report.align_last_two_elements(@current_section, title ) + end + def add_regression_plot( validation_set, name_attribute, section_title="Regression Plot", @@ -155,13 +160,13 @@ class Reports::ReportContent @xml_report.add_paragraph(section_regr, "No prediction info for regression available.") end end - - def add_roc_plot( validation_set, - split_set_attribute = nil, - section_title="ROC Plots", - section_text=nil, - image_titles=nil, - image_captions=nil) + + def add_roc_plot( validation_set, + accept_value, + split_set_attribute=nil, + image_title = "ROC Plot", + section_text=nil, + image_caption=nil) #section_roc = @xml_report.add_section(@current_section, section_title) section_roc = @current_section @@ -174,25 +179,18 @@ class Reports::ReportContent "validation set size: "+validation_set.size.to_s+", prediction set size: "+prediction_set.size.to_s end @xml_report.add_paragraph(section_roc, section_text) if section_text - - accept_values = validation_set.get_accept_values - accept_values.size.times do |i| - class_value = accept_values[i] - image_title = image_titles ? image_titles[i] : "ROC Plot for class-value '"+class_value.to_s+"'" - image_caption = image_captions ? image_captions[i] : nil - plot_file_name = "roc_plot"+@tmp_file_count.to_s+".png" - @tmp_file_count += 1 - begin - plot_file_path = add_tmp_file(plot_file_name) - Reports::PlotFactory.create_roc_plot( plot_file_path, prediction_set, class_value, split_set_attribute, false )#prediction_set.size>1 ) - @xml_report.add_imagefigure(section_roc, image_title, plot_file_name, "PNG", 100, image_caption) - rescue Exception => ex - msg = "WARNING could not create roc plot for class value '"+class_value.to_s+"': "+ex.message - LOGGER.error(msg) - rm_tmp_file(plot_file_name) - @xml_report.add_paragraph(section_roc, msg) - end - end + plot_file_name = "roc_plot"+@tmp_file_count.to_s+".png" + @tmp_file_count += 1 + begin + plot_file_path = add_tmp_file(plot_file_name) + Reports::PlotFactory.create_roc_plot( plot_file_path, prediction_set, accept_value, split_set_attribute )#prediction_set.size>1 ) + @xml_report.add_imagefigure(section_roc, image_title, plot_file_name, "PNG", 100, image_caption) + rescue Exception => ex + msg = "WARNING could not create roc plot for class value '"+accept_value.to_s+"': "+ex.message + LOGGER.error(msg) + rm_tmp_file(plot_file_name) + @xml_report.add_paragraph(section_roc, msg) + end else @xml_report.add_paragraph(section_roc, "No prediction-confidence info for roc plot available.") end @@ -200,11 +198,11 @@ class Reports::ReportContent end def add_confidence_plot( validation_set, + accept_value = nil, split_set_attribute = nil, - section_title="Confidence plots", + image_title = "Percent Correct vs Confidence Plot", section_text=nil, - image_titles=nil, - image_captions=nil) + image_caption=nil) #section_conf = @xml_report.add_section(@current_section, section_title) section_conf = @current_section @@ -217,30 +215,23 @@ class Reports::ReportContent "validation set size: "+validation_set.size.to_s+", prediction set size: "+prediction_set.size.to_s end @xml_report.add_paragraph(section_conf, section_text) if section_text - - image_title = image_titles ? image_titles[i] : "Percent Correct vs Confidence Plot" - image_caption = image_captions ? image_captions[i] : nil + plot_file_name = "conf_plot"+@tmp_file_count.to_s+".png" @tmp_file_count += 1 - begin - plot_file_path = add_tmp_file(plot_file_name) - Reports::PlotFactory.create_confidence_plot( plot_file_path, prediction_set, nil, split_set_attribute, false ) + Reports::PlotFactory.create_confidence_plot( plot_file_path, prediction_set, accept_value, split_set_attribute, false ) @xml_report.add_imagefigure(section_conf, image_title, plot_file_name, "PNG", 100, image_caption) - rescue Exception => ex msg = "WARNING could not create confidence plot: "+ex.message LOGGER.error(msg) rm_tmp_file(plot_file_name) @xml_report.add_paragraph(section_conf, msg) - end - + end else @xml_report.add_paragraph(section_conf, "No prediction-confidence info for confidence plot available.") end - - end + end def add_ranking_plots( validation_set, compare_attribute, diff --git a/report/report_factory.rb b/report/report_factory.rb index 7e74cb4..d16066e 100755 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -70,8 +70,12 @@ module Reports::ReportFactory report.add_result(validation_set, [:validation_uri] + VAL_ATTR_TRAIN_TEST + VAL_ATTR_CLASS, "Results", "Results") report.add_confusion_matrix(val) report.add_section("Plots") - report.add_roc_plot(validation_set) - report.add_confidence_plot(validation_set) + ([nil] + validation_set.get_accept_values).each do |accept_value| + report.add_roc_plot(validation_set, accept_value) + report.add_confidence_plot(validation_set, accept_value) + title = accept_value ? "Plots for predicted class-value '"+accept_value.to_s+"'" : "Plots for all predictions" + report.align_last_two_images title + end report.end_section when "regression" report.add_result(validation_set, [:validation_uri] + VAL_ATTR_TRAIN_TEST + VAL_ATTR_REGR, "Results", "Results") @@ -116,10 +120,15 @@ module Reports::ReportFactory report.add_result(cv_set, [:crossvalidation_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:crossvalidation_fold], res_titel, res_titel, res_text) report.add_confusion_matrix(cv_set.validations[0]) report.add_section("Plots") - report.add_roc_plot(validation_set) - report.add_roc_plot(validation_set, :crossvalidation_fold) - report.add_confidence_plot(validation_set) - report.add_confidence_plot(validation_set, :crossvalidation_fold) + [nil, :crossvalidation_fold].each do |split_attribute| + ([nil] + validation_set.get_accept_values).each do |accept_value| + report.add_roc_plot(validation_set, accept_value, split_attribute) + report.add_confidence_plot(validation_set, accept_value, split_attribute) + title = accept_value ? "Plots for predicted class-value '"+accept_value.to_s+"'" : "Plots for all predictions" + title += split_attribute ? ", separated by crossvalidation fold" : " (accumulated over all folds)" + report.align_last_two_images title + end + end report.end_section report.add_result(validation_set, [:validation_uri, :validation_report_uri]+VAL_ATTR_CV+VAL_ATTR_CLASS-[:num_folds, :dataset_uri, :algorithm_uri], "Results","Results") @@ -128,7 +137,7 @@ module Reports::ReportFactory report.add_section("Plots") report.add_regression_plot(validation_set, :crossvalidation_fold) report.add_confidence_plot(validation_set) - report.add_confidence_plot(validation_set, :crossvalidation_fold) + report.add_confidence_plot(validation_set, nil, :crossvalidation_fold) report.end_section report.add_result(validation_set, [:validation_uri, :validation_report_uri]+VAL_ATTR_CV+VAL_ATTR_REGR-[:num_folds, :dataset_uri, :algorithm_uri], "Results","Results") end diff --git a/report/xml_report.rb b/report/xml_report.rb index 7467c47..b308c01 100755 --- a/report/xml_report.rb +++ b/report/xml_report.rb @@ -93,38 +93,70 @@ module Reports end end - # adds a new image to a REXML:Element, returns the figure as element - # - # example: add_imagefigure( section2, "Nice graph", "/images/graph1.svg", "SVG", "This graph shows..." ) - # - # call-seq: - # add_imagefigure( element, title, path, filetype, caption = nil ) => REXML::Element - # - def add_imagefigure( element, title, path, filetype, size_pct=100, caption = nil ) - + def imagefigure( title, path, filetype, size_pct=100, caption = nil ) figure = Reports::XMLReportUtil.attribute_element("figure", {"float" => 0}) figure << Reports::XMLReportUtil.text_element("title", title) media = Element.new("mediaobject") image = Element.new("imageobject") imagedata = Reports::XMLReportUtil.attribute_element("imagedata", - {"fileref" => path, "format"=>filetype, "contentwidth" => size_pct.to_s+"%", - #"contentdepth"=> "4in" - })#"width" => "6in", "height" => "5in"}) #"contentwidth" => "100%"}) + {"fileref" => path, "format"=>filetype, "contentwidth" => size_pct.to_s+"%", + #"contentdepth"=> "4in" + })#"width" => "6in", "height" => "5in"}) #"contentwidth" => "100%"}) #imagedata = Reports::XMLReportUtil.attribute_element("imagedata",{"width" => "6in", "fileref" => path, "format"=>filetype}) @resource_path_elements[imagedata] = "fileref" image << imagedata - media << image - # ulink = Element.new("ulink") # ulink.add_attributes({"url" => "http://google.de"}) # ulink << image # media << ulink - media << Reports::XMLReportUtil.text_element("caption", caption) if caption - figure << media + figure << media + figure + end + + # adds a new image to a REXML:Element, returns the figure as element + # + # example: add_imagefigure( section2, "Nice graph", "/images/graph1.svg", "SVG", "This graph shows..." ) + # + # call-seq: + # add_imagefigure( element, title, path, filetype, caption = nil ) => REXML::Element + # + def add_imagefigure( element, title, path, filetype, size_pct=100, caption = nil ) + figure = imagefigure( title, path, filetype, size_pct, caption) element << figure - return figure + return figure + end + + # bit of a hack to algin the last two elements that have been added to element into one row + def align_last_two_elements( element, title ) + imgs = [] + element.elements.each do |e| + imgs[0] = imgs[1] + imgs[1] = e + end + element.delete_element imgs[0] + element.delete_element imgs[1] + add_imagefigures_in_row( element, imgs, title ) + end + + def add_imagefigures_in_row( element, imagefigures, title ) + params = {"frame" => "none", "colsep" => 0, "rowsep" => 0 } + table = Reports::XMLReportUtil.attribute_element("table",params) + table << Reports::XMLReportUtil.text_element("title", title) + tgroup = Reports::XMLReportUtil.attribute_element("tgroup",{"cols" => 2}) + tbody = Element.new("tbody") + row = Element.new("row") + imagefigures.each do |f| + entry = Element.new("entry") + entry << f + row << entry + end + tbody << row + tgroup << tbody + table << tgroup + element << table + table end def add_image( element, url ) -- cgit v1.2.3 From 750c3a57db4ed5ad3ec50c7875ad3edf877727f6 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 20 May 2011 10:51:18 +0200 Subject: handle confidence nil values, and fix prediction confidence sorting, improved prediction table --- lib/ot_predictions.rb | 19 +++++++++++-------- report/environment.rb | 2 ++ report/report_application.rb | 4 ++++ report/xml_report.rb | 10 +++++----- resources/error.png | Bin 0 -> 1060 bytes resources/ok.png | Bin 0 -> 976 bytes resources/ot-logo.png | Bin 0 -> 14338 bytes 7 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 resources/error.png create mode 100644 resources/ok.png create mode 100644 resources/ot-logo.png diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index 644168f..93581d8 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -146,11 +146,11 @@ module Lib predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable) end # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable) - conf = predicted_values[count]!=nil ? 1 : 0 + conf = predicted_values[count]!=nil ? 1 : nil begin feature = prediction_dataset.data_entries[c].keys[0] feature_data = prediction_dataset.features[feature] - conf = feature_data[OT.confidence] if feature_data[OT.confidence]!=nil + conf = feature_data[OT.confidence].to_f if feature_data[OT.confidence]!=nil rescue LOGGER.warn "could not get confidence" end @@ -259,13 +259,17 @@ module Lib a << (format ? p.predicted_value(i).to_nice_s : p.predicted_value(i)) if p.feature_type=="classification" if (p.predicted_value(i)!=nil and p.actual_value(i)!=nil) - a << (p.classification_miss?(i) ? 1 : 0) + if p.classification_miss?(i) + a << (format ? ICON_ERROR : 1) + else + a << (format ? ICON_OK : 0) + end else a << nil end end if p.confidence_values_available? - conf_column = a.size + conf_column = a.size if conf_column==nil a << p.confidence_value(i) #(format ? p.confidence_value(i).to_nice_s : p.confidence_value(i)) end a << p.identifier(i) @@ -274,10 +278,10 @@ module Lib end if conf_column!=nil - res.sort!{ |x,y| y[4] <=> x[4] } + res = res.sort_by{ |n| n[conf_column] || 0 }.reverse if format res.each do |a| - a[4] = a[4].to_nice_s + a[conf_column] = a[conf_column].to_nice_s end end end @@ -291,7 +295,6 @@ module Lib res.insert(0, header) return res - end - + end end end diff --git a/report/environment.rb b/report/environment.rb index 59465aa..bdb4a04 100755 --- a/report/environment.rb +++ b/report/environment.rb @@ -27,6 +27,8 @@ require "report/validation_data.rb" require "report/util.rb" require "report/statistical_test.rb" +ICON_ERROR = File.join(CONFIG[:services]["opentox-validation"],"resources/error.png") +ICON_OK = File.join(CONFIG[:services]["opentox-validation"],"resources/ok.png") diff --git a/report/report_application.rb b/report/report_application.rb index 1637ead..3c8670a 100755 --- a/report/report_application.rb +++ b/report/report_application.rb @@ -23,6 +23,10 @@ get '/'+ENV['DOCBOOK_DIRECTORY']+'/:resource' do get_docbook_resource ENV['DOCBOOK_DIRECTORY']+"/"+request.env['REQUEST_URI'].split("/")[-1] end +get '/resources/:resource' do + get_docbook_resource "resources/"+request.env['REQUEST_URI'].split("/")[-1] +end + get '/report/:type/css_style_sheet/?' do perform do |rs| "@import \""+params[:css_style_sheet]+"\";" diff --git a/report/xml_report.rb b/report/xml_report.rb index b308c01..cf30e3b 100755 --- a/report/xml_report.rb +++ b/report/xml_report.rb @@ -159,16 +159,16 @@ module Reports table end - def add_image( element, url ) + def add_image( element, url ) #, scale=false ) image = Element.new("imageobject") - imagedata = Reports::XMLReportUtil.attribute_element("imagedata", - {"fileref" => url, "format"=>"PNG", "contentwidth" => "2in" }) #PENDING: do not hardcode size + params = {"fileref" => url, "format"=>"PNG"} + #params["contentwidth"] = "2in" + imagedata = Reports::XMLReportUtil.attribute_element("imagedata",params) image << imagedata element << image return image end - # adds a table to a REXML:Element, _table_values_ should be a multi-dimensional-array, returns the table as element # # call-seq: @@ -216,7 +216,7 @@ module Reports row = Element.new("row") r.each do |v| entry = Element.new("entry") - if auto_link_urls && v.to_s =~ /depict/ || v.to_s =~ /image\/png$/ #PENDING + if auto_link_urls && v.to_s =~ /depict/ || v.to_s =~ /png$/ #PENDING add_image(entry, v.to_s) elsif auto_link_urls && v.to_s =~ /^http(s?):\/\// #add_url(entry, v.to_s, v.to_s) diff --git a/resources/error.png b/resources/error.png new file mode 100644 index 0000000..e051534 Binary files /dev/null and b/resources/error.png differ diff --git a/resources/ok.png b/resources/ok.png new file mode 100644 index 0000000..31bd433 Binary files /dev/null and b/resources/ok.png differ diff --git a/resources/ot-logo.png b/resources/ot-logo.png new file mode 100644 index 0000000..248a853 Binary files /dev/null and b/resources/ot-logo.png differ -- cgit v1.2.3 From 452f119ae9e15ac51c4ed09367b6eab9f4392beb Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 20 May 2011 12:44:41 +0200 Subject: for new predictions in reports: fix naming --- lib/ot_predictions.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index 93581d8..6b90de3 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -289,7 +289,7 @@ module Lib header << "compound" if add_pic header << "actual value" header << "predicted value" - header << "missclassified" if predictions[0].feature_type=="classification" + header << "classification" if predictions[0].feature_type=="classification" header << "confidence value" if predictions[0].confidence_values_available? header << "compound-uri" res.insert(0, header) -- cgit v1.2.3 From 86a1d1e5fda994bc318100963b5fe7af3da24a53 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 23 May 2011 10:51:29 +0200 Subject: new feature: zoom into plots; (png-)plots in reports link to identical svg-plots --- report/plot_factory.rb | 47 +++++++++++++++++++++++++++++------------------ report/report_content.rb | 44 ++++++++++++++++++++------------------------ report/xml_report.rb | 25 +++++++++++++++---------- 3 files changed, 64 insertions(+), 52 deletions(-) diff --git a/report/plot_factory.rb b/report/plot_factory.rb index 7535eb4..2a621f4 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -52,9 +52,10 @@ module Reports module PlotFactory - def self.create_regression_plot( out_file, validation_set, name_attribute ) + def self.create_regression_plot( out_files, validation_set, name_attribute ) - LOGGER.debug "Creating regression plot, out-file:"+out_file.to_s + out_files = [out_files] unless out_files.is_a?(Array) + LOGGER.debug "Creating regression plot, out-file:"+out_files.to_s names = [] x = [] @@ -79,7 +80,9 @@ module Reports end raise "no predictions performed" if x.size==0 || x[0].size==0 - RubyPlot::regression_point_plot(out_file, "Regression plot", "Predicted values", "Actual values", names, x, y ) + out_files.each do |out_file| + RubyPlot::regression_point_plot(out_file, "Regression plot", "Predicted values", "Actual values", names, x, y ) + end end @@ -91,10 +94,11 @@ module Reports # * the validation set is splitted into sets of validation_sets with equal attribute values # * each of theses validation sets is plotted as a roc-curve # - def self.create_roc_plot( out_file, validation_set, class_value, split_set_attribute=nil, + def self.create_roc_plot( out_files, validation_set, class_value, split_set_attribute=nil, x_label="False positive rate", y_label="True Positive Rate", show_single_curves=false ) - LOGGER.debug "creating roc plot for '"+validation_set.size.to_s+"' validations, out-file:"+out_file.to_s + out_files = [out_files] unless out_files.is_a?(Array) + LOGGER.debug "creating roc plot for '"+validation_set.size.to_s+"' validations, out-files:"+out_files.inspect if split_set_attribute attribute_values = validation_set.get_values(split_set_attribute) @@ -120,14 +124,17 @@ module Reports labels << ["confidence: "+confidence.to_nice_s, point[0], point[1]] end end - RubyPlot::plot_lines(out_file, "ROC-Plot", x_label, y_label, data[:names], data[:fp_rate], data[:tp_rate], data[:faint], labels ) + out_files.each do |out_file| + RubyPlot::plot_lines(out_file, "ROC-Plot", x_label, y_label, data[:names], data[:fp_rate], data[:tp_rate], data[:faint], labels ) + end end end - def self.create_confidence_plot( out_file, validation_set, class_value, split_set_attribute=nil, show_single_curves=false ) + def self.create_confidence_plot( out_files, validation_set, class_value, split_set_attribute=nil, show_single_curves=false ) - LOGGER.debug "creating confidence plot for '"+validation_set.size.to_s+"' validations, out-file:"+out_file.to_s + out_files = [out_files] unless out_files.is_a?(Array) + LOGGER.debug "creating confidence plot for '"+validation_set.size.to_s+"' validations, out-file:"+out_files.inspect if split_set_attribute attribute_values = validation_set.get_values(split_set_attribute) @@ -145,19 +152,23 @@ module Reports end end #RubyPlot::plot_lines(out_file, "Percent Correct vs Confidence Plot", "Confidence", "Percent Correct", names, fp_rates, tp_rates ) - case validation_set.unique_feature_type - when "classification" - RubyPlot::accuracy_confidence_plot(out_file, "Percent Correct vs Confidence Plot", "Confidence", "Percent Correct", names, confidence, performance) - when "regression" - RubyPlot::accuracy_confidence_plot(out_file, "RMSE vs Confidence Plot", "Confidence", "RMSE", names, confidence, performance, true) + out_files.each do |out_file| + case validation_set.unique_feature_type + when "classification" + RubyPlot::accuracy_confidence_plot(out_file, "Percent Correct vs Confidence Plot", "Confidence", "Percent Correct", names, confidence, performance) + when "regression" + RubyPlot::accuracy_confidence_plot(out_file, "RMSE vs Confidence Plot", "Confidence", "RMSE", names, confidence, performance, true) + end end else data = transform_confidence_predictions(validation_set, class_value, show_single_curves) - case validation_set.unique_feature_type - when "classification" - RubyPlot::accuracy_confidence_plot(out_file, "Percent Correct vs Confidence Plot", "Confidence", "Percent Correct", data[:names], data[:confidence], data[:performance]) - when "regression" - RubyPlot::accuracy_confidence_plot(out_file, "RMSE vs Confidence Plot", "Confidence", "RMSE", data[:names], data[:confidence], data[:performance], true) + out_files.each do |out_file| + case validation_set.unique_feature_type + when "classification" + RubyPlot::accuracy_confidence_plot(out_file, "Percent Correct vs Confidence Plot", "Confidence", "Percent Correct", data[:names], data[:confidence], data[:performance]) + when "regression" + RubyPlot::accuracy_confidence_plot(out_file, "RMSE vs Confidence Plot", "Confidence", "RMSE", data[:names], data[:confidence], data[:performance], true) + end end end end diff --git a/report/report_content.rb b/report/report_content.rb index 893ac34..a8b700b 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -133,8 +133,7 @@ class Reports::ReportContent name_attribute, section_title="Regression Plot", section_text=nil, - image_title=nil, - image_caption=nil) + image_title=nil) image_title = "Regression plot" unless image_title #section_regr = @xml_report.add_section(@current_section, section_title) @@ -145,12 +144,11 @@ class Reports::ReportContent section_text += "\nWARNING: regression plot information not available for all validation results" if prediction_set.size!=validation_set.size @xml_report.add_paragraph(section_regr, section_text) if section_text - plot_file_name = "regr_plot"+@tmp_file_count.to_s+".png" - @tmp_file_count += 1 begin - plot_file_path = add_tmp_file(plot_file_name) - Reports::PlotFactory.create_regression_plot( plot_file_path, prediction_set, name_attribute ) - @xml_report.add_imagefigure(section_regr, image_title, plot_file_name, "PNG", 100, image_caption) + plot_png = add_tmp_file("regr_plot", "png") + plot_svg = add_tmp_file("regr_plot", "svg") + Reports::PlotFactory.create_regression_plot( [plot_png[:path], plot_svg[:path]], prediction_set, name_attribute ) + @xml_report.add_imagefigure(section_regr, image_title, plot_png[:name], "PNG", 100, plot_svg[:name]) rescue Exception => ex LOGGER.error("Could not create regression plot: "+ex.message) rm_tmp_file(plot_file_name) @@ -179,15 +177,14 @@ class Reports::ReportContent "validation set size: "+validation_set.size.to_s+", prediction set size: "+prediction_set.size.to_s end @xml_report.add_paragraph(section_roc, section_text) if section_text - plot_file_name = "roc_plot"+@tmp_file_count.to_s+".png" - @tmp_file_count += 1 begin - plot_file_path = add_tmp_file(plot_file_name) - Reports::PlotFactory.create_roc_plot( plot_file_path, prediction_set, accept_value, split_set_attribute )#prediction_set.size>1 ) - @xml_report.add_imagefigure(section_roc, image_title, plot_file_name, "PNG", 100, image_caption) + plot_png = add_tmp_file("roc_plot", "png") + plot_svg = add_tmp_file("roc_plot", "svg") + Reports::PlotFactory.create_roc_plot( [plot_png[:path], plot_svg[:path]], prediction_set, accept_value, split_set_attribute )#prediction_set.size>1 ) + @xml_report.add_imagefigure(section_roc, image_title, plot_png[:name], "PNG", 100, plot_svg[:name]) rescue Exception => ex msg = "WARNING could not create roc plot for class value '"+accept_value.to_s+"': "+ex.message - LOGGER.error(msg) + #LOGGER.error(msg) rm_tmp_file(plot_file_name) @xml_report.add_paragraph(section_roc, msg) end @@ -201,8 +198,7 @@ class Reports::ReportContent accept_value = nil, split_set_attribute = nil, image_title = "Percent Correct vs Confidence Plot", - section_text=nil, - image_caption=nil) + section_text="") #section_conf = @xml_report.add_section(@current_section, section_title) section_conf = @current_section @@ -214,14 +210,13 @@ class Reports::ReportContent LOGGER.error "WARNING: plot information not available for all validation results:\n"+ "validation set size: "+validation_set.size.to_s+", prediction set size: "+prediction_set.size.to_s end - @xml_report.add_paragraph(section_conf, section_text) if section_text + @xml_report.add_paragraph(section_conf, section_text) if section_text and section_text.size>0 - plot_file_name = "conf_plot"+@tmp_file_count.to_s+".png" - @tmp_file_count += 1 begin - plot_file_path = add_tmp_file(plot_file_name) - Reports::PlotFactory.create_confidence_plot( plot_file_path, prediction_set, accept_value, split_set_attribute, false ) - @xml_report.add_imagefigure(section_conf, image_title, plot_file_name, "PNG", 100, image_caption) + plot_png = add_tmp_file("conf_plot", "png") + plot_svg = add_tmp_file("conf_plot", "svg") + Reports::PlotFactory.create_confidence_plot( [plot_png[:path], plot_svg[:path]], prediction_set, accept_value, split_set_attribute, false ) + @xml_report.add_imagefigure(section_conf, image_title, plot_png[:name], "PNG", 100, plot_svg[:name]) rescue Exception => ex msg = "WARNING could not create confidence plot: "+ex.message LOGGER.error(msg) @@ -298,13 +293,14 @@ class Reports::ReportContent end private - def add_tmp_file(tmp_file_name) - + def add_tmp_file(name, extension) + tmp_file_name = name.to_s+@tmp_file_count.to_s+"."+extension.to_s + @tmp_file_count += 1 @tmp_files = {} unless @tmp_files raise "file name already exits" if @tmp_files[tmp_file_name] || (@text_files && @text_files[tmp_file_name]) tmp_file_path = Reports::Util.create_tmp_file(tmp_file_name) @tmp_files[tmp_file_name] = tmp_file_path - return tmp_file_path + return {:name => tmp_file_name, :path => tmp_file_path} end def rm_tmp_file(tmp_file_name) diff --git a/report/xml_report.rb b/report/xml_report.rb index cf30e3b..855a65e 100755 --- a/report/xml_report.rb +++ b/report/xml_report.rb @@ -93,10 +93,12 @@ module Reports end end - def imagefigure( title, path, filetype, size_pct=100, caption = nil ) + def imagefigure( title, path, filetype, size_pct=100, altPath = nil ) figure = Reports::XMLReportUtil.attribute_element("figure", {"float" => 0}) figure << Reports::XMLReportUtil.text_element("title", title) - media = Element.new("mediaobject") + + #media = Element.new("mediaobject") + media = Element.new("inlinemediaobject") image = Element.new("imageobject") imagedata = Reports::XMLReportUtil.attribute_element("imagedata", {"fileref" => path, "format"=>filetype, "contentwidth" => size_pct.to_s+"%", @@ -106,12 +108,15 @@ module Reports @resource_path_elements[imagedata] = "fileref" image << imagedata media << image -# ulink = Element.new("ulink") -# ulink.add_attributes({"url" => "http://google.de"}) -# ulink << image -# media << ulink - media << Reports::XMLReportUtil.text_element("caption", caption) if caption - figure << media + #media << Reports::XMLReportUtil.text_element("caption", caption) if caption + #figure << media + + ulink = Element.new("ulink") + ulink.add_attributes({"url" => altPath ? altPath : path }) + @resource_path_elements[ulink] = "url" + ulink << media + + figure << ulink figure end @@ -122,8 +127,8 @@ module Reports # call-seq: # add_imagefigure( element, title, path, filetype, caption = nil ) => REXML::Element # - def add_imagefigure( element, title, path, filetype, size_pct=100, caption = nil ) - figure = imagefigure( title, path, filetype, size_pct, caption) + def add_imagefigure( element, title, path, filetype, size_pct=100, altPath = nil ) + figure = imagefigure( title, path, filetype, size_pct, altPath) element << figure return figure end -- cgit v1.2.3 From 63389a0844313428899cbdb3d7cda4042fd0adc1 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 23 May 2011 10:59:34 +0200 Subject: fix for plot-zoom feature --- report/plot_factory.rb | 4 +++- report/report_content.rb | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/report/plot_factory.rb b/report/plot_factory.rb index 2a621f4..dfb1369 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -115,7 +115,9 @@ module Reports LOGGER.warn "could not create ROC plot for "+value.to_s end end - RubyPlot::plot_lines(out_file, "ROC-Plot", x_label, y_label, names, fp_rates, tp_rates ) + out_files.each do |out_file| + RubyPlot::plot_lines(out_file, "ROC-Plot", x_label, y_label, names, fp_rates, tp_rates ) + end else data = transform_roc_predictions(validation_set, class_value, show_single_curves) labels = [] diff --git a/report/report_content.rb b/report/report_content.rb index a8b700b..d60f700 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -151,7 +151,8 @@ class Reports::ReportContent @xml_report.add_imagefigure(section_regr, image_title, plot_png[:name], "PNG", 100, plot_svg[:name]) rescue Exception => ex LOGGER.error("Could not create regression plot: "+ex.message) - rm_tmp_file(plot_file_name) + rm_tmp_file(plot_png[:name]) + rm_tmp_file(plot_svg[:name]) @xml_report.add_paragraph(section_regr, "could not create regression plot: "+ex.message) end else @@ -184,8 +185,9 @@ class Reports::ReportContent @xml_report.add_imagefigure(section_roc, image_title, plot_png[:name], "PNG", 100, plot_svg[:name]) rescue Exception => ex msg = "WARNING could not create roc plot for class value '"+accept_value.to_s+"': "+ex.message - #LOGGER.error(msg) - rm_tmp_file(plot_file_name) + LOGGER.error(msg) + rm_tmp_file(plot_png[:name]) + rm_tmp_file(plot_svg[:name]) @xml_report.add_paragraph(section_roc, msg) end else @@ -220,7 +222,8 @@ class Reports::ReportContent rescue Exception => ex msg = "WARNING could not create confidence plot: "+ex.message LOGGER.error(msg) - rm_tmp_file(plot_file_name) + rm_tmp_file(plot_png[:name]) + rm_tmp_file(plot_svg[:name]) @xml_report.add_paragraph(section_conf, msg) end else -- cgit v1.2.3 From 9105a8cd315ae0eb220af8879d4a83ac2fbe3740 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 23 May 2011 16:35:42 +0200 Subject: fix subjectid bug when creating cv stats --- validation/validation_service.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 520d84c..e0c520e 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -40,14 +40,14 @@ module Validation raise OpenTox::BadRequestError.new "Crossvalidation '"+cv_id.to_s+"' not finished" unless crossvalidation.finished vals = Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|x| x} - feature_type = OpenTox::Model::Generic.new(vals.first.model_uri).feature_type(@subjectid) + feature_type = OpenTox::Model::Generic.new(vals.first.model_uri).feature_type(subjectid) test_dataset_uris = vals.collect{|v| v.test_dataset_uri} test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri} prediction_feature = vals.first.prediction_feature prediction_dataset_uris = vals.collect{|v| v.prediction_dataset_uri} predicted_variables = vals.collect{|v| nil} prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, - prediction_dataset_uris, predicted_variables, @subjectid ) + prediction_dataset_uris, predicted_variables, subjectid ) v = Validation.new case feature_type -- cgit v1.2.3 From 4f15bab6c8598b81a5514fdb631834b637bf1ef0 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 23 May 2011 16:49:50 +0200 Subject: fix read predicted variables in crossvaldation --- validation/validation_service.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index e0c520e..f713120 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -40,12 +40,13 @@ module Validation raise OpenTox::BadRequestError.new "Crossvalidation '"+cv_id.to_s+"' not finished" unless crossvalidation.finished vals = Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|x| x} - feature_type = OpenTox::Model::Generic.new(vals.first.model_uri).feature_type(subjectid) + models = vals.collect{|v| OpenTox::Model::Generic.find(v.model_uri)} + feature_type = models.first.feature_type(subjectid) test_dataset_uris = vals.collect{|v| v.test_dataset_uri} test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri} prediction_feature = vals.first.prediction_feature prediction_dataset_uris = vals.collect{|v| v.prediction_dataset_uri} - predicted_variables = vals.collect{|v| nil} + predicted_variables = models.collect{|m| m.metadata[OT.predictedVariables]} prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, prediction_dataset_uris, predicted_variables, subjectid ) -- cgit v1.2.3 From 51aeb71240ba93972b09c9e42845a6f8a72103d2 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 23 May 2011 16:58:16 +0200 Subject: fix read predicted variables in crossvaldation: forgot to add subjectid --- validation/validation_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index f713120..61f3a6e 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -40,7 +40,7 @@ module Validation raise OpenTox::BadRequestError.new "Crossvalidation '"+cv_id.to_s+"' not finished" unless crossvalidation.finished vals = Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation" ).collect{|x| x} - models = vals.collect{|v| OpenTox::Model::Generic.find(v.model_uri)} + models = vals.collect{|v| OpenTox::Model::Generic.find(v.model_uri, subjectid)} feature_type = models.first.feature_type(subjectid) test_dataset_uris = vals.collect{|v| v.test_dataset_uri} test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri} -- cgit v1.2.3 From d67f6cd4e1242fce926632c948035d2776745273 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 23 May 2011 17:09:32 +0200 Subject: fix: set stratified to false if "false"||"0"||" --- validation/validation_application.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 7db2a6a..d448c62 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -36,7 +36,8 @@ post '/crossvalidation/?' do cv_params = { :dataset_uri => params[:dataset_uri], :algorithm_uri => params[:algorithm_uri] } - [ :num_folds, :random_seed, :stratified ].each{ |sym| cv_params[sym] = params[sym] if params[sym] } + [ :num_folds, :random_seed ].each{ |sym| cv_params[sym] = params[sym] if params[sym] } + cv_params[:stratified] = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] cv = Validation::Crossvalidation.create cv_params cv.subjectid = @subjectid cv.perform_cv( params[:prediction_feature], params[:algorithm_params], task ) -- cgit v1.2.3 From eeb4f570f8e991b77ae5e9a7d9a58cc6b2dd115a Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 24 May 2011 09:59:02 +0200 Subject: fix r-square computation --- lib/predictions.rb | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/predictions.rb b/lib/predictions.rb index f97b764..4a15e1d 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -481,7 +481,12 @@ module Lib end def r_square - return sample_correlation_coefficient ** 2 + #return sample_correlation_coefficient ** 2 + + # see http://en.wikipedia.org/wiki/Coefficient_of_determination#Definitions + # see http://web.maths.unsw.edu.au/~adelle/Garvan/Assays/GoodnessOfFit.html + r_2 = 1 - residual_sum_of_squares / total_sum_of_squares + ( r_2.infinite? || r_2.nan? ) ? 0 : r_2 end def sample_correlation_coefficient @@ -493,7 +498,16 @@ module Lib end def total_sum_of_squares - return @variance_actual * ( @num_predicted - 1 ) + #return @variance_actual * ( @num_predicted - 1 ) + sum = 0 + @predicted_values.size.times do |i| + sum += (@actual_values[i]-@actual_mean)**2 if @predicted_values[i]!=nil + end + sum + end + + def residual_sum_of_squares + sum_squared_error end def target_variance_predicted -- cgit v1.2.3 From 2317a75d3e91a6a03992a8f19d6559323146a256 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 24 May 2011 10:19:33 +0200 Subject: extended error msg when too many predictions (for pantelis) --- lib/ot_predictions.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index 6b90de3..2e3c238 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -121,8 +121,9 @@ module Lib end end - raise "more predicted than test compounds test:"+compounds.size.to_s+" < prediction:"+ - prediction_dataset.compounds.size.to_s if compounds.size < prediction_dataset.compounds.size + raise "more predicted than test compounds, #test: "+compounds.size.to_s+" < #prediction: "+ + prediction_dataset.compounds.size.to_s+", test-dataset: "+test_dataset_uri.to_s+", prediction-dataset: "+ + prediction_dataset_uri if compounds.size < prediction_dataset.compounds.size if CHECK_VALUES prediction_dataset.compounds.each do |c| raise "predicted compound not found in test dataset:\n"+c+"\ntest-compounds:\n"+ -- cgit v1.2.3 From 1c2ed2e04d5cbd1842e646e6429bcfdc0b1372dd Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 24 May 2011 16:14:37 +0200 Subject: rewrite: adjust to new prediction dataset format from lazar, read confidence from ambit algorithms --- lib/feature_util.rb | 66 ++++++++++++++++++++++++++++ lib/ot_predictions.rb | 83 ++++++++++++++++++------------------ lib/predictions.rb | 27 ++++++------ report/report_content.rb | 9 ++-- report/validation_access.rb | 20 ++++++--- report/validation_data.rb | 5 +++ validation/validation_application.rb | 10 +++-- validation/validation_service.rb | 22 +++++++--- 8 files changed, 163 insertions(+), 79 deletions(-) create mode 100644 lib/feature_util.rb diff --git a/lib/feature_util.rb b/lib/feature_util.rb new file mode 100644 index 0000000..405d991 --- /dev/null +++ b/lib/feature_util.rb @@ -0,0 +1,66 @@ + +module Lib + class FeatureUtil + + # this derieves the predicted_variable and predicted_confidence in prediction dataset + # predicted_variable: the feature-uri of model predictions + # predicted_confidence: the feature-uri of the model prediction confidence + # according to API info should be available in the API + # problem: IST has no feature-service -> predicted_variable depends on prediction dataset + # + # PENDING: ambit and ist declare prediction features differently -> unify + # + def self.predicted_variables( model, prediction_dataset_uri, subjectid=nil ) + + predicted_variable = nil + predicted_confidence = nil + + if model.metadata[OT.predictedVariables] + predictedVariables = model.metadata[OT.predictedVariables] + if predictedVariables.is_a?(Array) + if (predictedVariables.size==1) + predicted_variable = predictedVariables[0] + elsif (predictedVariables.size==2) + # PENDING identify confidence + conf_index = -1 + predictedVariables.size.times do |i| + conf_index = i if OpenTox::Feature.find(predictedVariables[i]).metadata[DC.title]=~/(?i)confidence/ + end + raise "size=2, no confidence "+predictedVariables.inspect+" "+model.uri.to_s if conf_index==-1 + predicted_variable = predictedVariables[1-conf_index] + predicted_confidence = predictedVariables[conf_index] + else + raise "size>2 "+predictedVariables.inspect+" "+model.uri.to_s + end + else + raise "predictedVariables is no array" + end + end + + unless predicted_variable + d = OpenTox::Dataset.new prediction_dataset_uri + d.load_features(subjectid) + d.features.keys.each do |f| + if d.features[f][OT.hasSource]==model.uri + puts "source matching" + # PENDING identify confidence + if f =~ /(?i)confidence/ + puts "conf matiching" + raise "duplicate confidence feature, what to choose?" if predicted_confidence!=nil + predicted_confidence = f + elsif d.features[f][RDF.type].include? OT.ModelPrediction + puts "type include prediction" + raise "duplicate predicted variable, what to choose?" if predicted_variable!=nil + predicted_variable = f + end + end + puts d.features[f][OT.hasSource] + end + raise "could not estimate predicted variable" unless predicted_variable + end + + {:predicted_variable => predicted_variable, :predicted_confidence => predicted_confidence} + end + end +end + diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index 2e3c238..df1efc3 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -16,16 +16,18 @@ module Lib end def initialize( feature_type, test_dataset_uris, test_target_dataset_uris, - prediction_feature, prediction_dataset_uris, predicted_variables, subjectid=nil, task=nil) + prediction_feature, prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid=nil, task=nil) test_dataset_uris = [test_dataset_uris] unless test_dataset_uris.is_a?(Array) test_target_dataset_uris = [test_target_dataset_uris] unless test_target_dataset_uris.is_a?(Array) prediction_dataset_uris = [prediction_dataset_uris] unless prediction_dataset_uris.is_a?(Array) predicted_variables = [predicted_variables] unless predicted_variables.is_a?(Array) + predicted_confidences = [predicted_confidences] unless predicted_confidences.is_a?(Array) LOGGER.debug "loading prediciton -- test-dataset: "+test_dataset_uris.inspect LOGGER.debug "loading prediciton -- test-target-datset: "+test_target_dataset_uris.inspect LOGGER.debug "loading prediciton -- prediction-dataset: "+prediction_dataset_uris.inspect LOGGER.debug "loading prediciton -- predicted_variable: "+predicted_variables.inspect + LOGGER.debug "loading prediciton -- predicted_confidence: "+predicted_confidences.inspect LOGGER.debug "loading prediciton -- prediction_feature: "+prediction_feature.to_s raise "prediction_feature missing" unless prediction_feature @@ -46,6 +48,7 @@ module Lib test_target_dataset_uri = test_target_dataset_uris[i] prediction_dataset_uri = prediction_dataset_uris[i] predicted_variable = predicted_variables[i] + predicted_confidence = predicted_confidences[i] predicted_variable=prediction_feature if predicted_variable==nil @@ -72,7 +75,7 @@ module Lib "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil end - + compounds = test_dataset.compounds LOGGER.debug "test dataset size: "+compounds.size.to_s raise "test dataset is empty "+test_dataset_uri.to_s unless compounds.size>0 @@ -92,35 +95,24 @@ module Lib compounds.each do |c| case feature_type when "classification" - actual_values << classification_value(test_target_dataset, c, prediction_feature, accept_values) + actual_values << classification_val(test_target_dataset, c, prediction_feature, accept_values) when "regression" - actual_values << regression_value(test_target_dataset, c, prediction_feature) + actual_values << regression_val(test_target_dataset, c, prediction_feature) end end task.progress( task_status += task_step ) if task # loaded actual values prediction_dataset = Lib::DatasetCache.find prediction_dataset_uri,subjectid raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset - - # TODO: remove LAZAR_PREDICTION_DATASET_HACK - no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil - if no_prediction_feature - one_entry_per_compound = true - compounds.each do |c| - if prediction_dataset.data_entries[c] and prediction_dataset.data_entries[c].size != 1 - one_entry_per_compound = false - break - end - end - msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+ - prediction_dataset.features.keys.inspect - if one_entry_per_compound - LOGGER.warn msg - else - raise msg - end - end - + raise "predicted_variable not found in prediction_dataset\n"+ + "predicted_variable '"+predicted_variable.to_s+"'\n"+ + "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+ + "available features are: "+prediction_dataset.features.inspect if prediction_dataset.features.keys.index(predicted_variable)==nil + raise "predicted_confidence not found in prediction_dataset\n"+ + "predicted_confidence '"+predicted_confidence.to_s+"'\n"+ + "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+ + "available features are: "+prediction_dataset.features.inspect if predicted_confidence and prediction_dataset.features.keys.index(predicted_confidence)==nil + raise "more predicted than test compounds, #test: "+compounds.size.to_s+" < #prediction: "+ prediction_dataset.compounds.size.to_s+", test-dataset: "+test_dataset_uri.to_s+", prediction-dataset: "+ prediction_dataset_uri if compounds.size < prediction_dataset.compounds.size @@ -141,25 +133,18 @@ module Lib else case feature_type when "classification" - # TODO: remove LAZAR_PREDICTION_DATASET_HACK - predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, accept_values) + predicted_values << classification_val(prediction_dataset, c, predicted_variable, accept_values) when "regression" - predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable) + predicted_values << regression_val(prediction_dataset, c, predicted_variable) end - # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable) - conf = predicted_values[count]!=nil ? 1 : nil - begin - feature = prediction_dataset.data_entries[c].keys[0] - feature_data = prediction_dataset.features[feature] - conf = feature_data[OT.confidence].to_f if feature_data[OT.confidence]!=nil - rescue - LOGGER.warn "could not get confidence" + if predicted_confidence + confidence_values << confidence_val(prediction_dataset, c, predicted_confidence) + else + confidence_values << nil end - confidence_values << conf end count += 1 end - @compounds += compounds all_predicted_values += predicted_values all_actual_values += actual_values @@ -174,7 +159,7 @@ module Lib end private - def regression_value(dataset, compound, feature) + def regression_val(dataset, compound, feature) v = value(dataset, compound, feature) begin v = v.to_f unless v==nil or v.is_a?(Numeric) @@ -185,7 +170,18 @@ module Lib end end - def classification_value(dataset, compound, feature, accept_values) + def confidence_val(dataset, compound, confidence) + v = value(dataset, compound, confidence) + begin + v = v.to_f unless v==nil or v.is_a?(Numeric) + v + rescue + LOGGER.warn "no numeric value for confidence '"+v.to_s+"'" + nil + end + end + + def classification_val(dataset, compound, feature, accept_values) v = value(dataset, compound, feature) i = accept_values.index(v.to_s) raise "illegal class_value of prediction (value is '"+v.to_s+"'), accept values are "+ @@ -239,6 +235,10 @@ module Lib def self.to_array( predictions, add_pic=false, format=false ) + confidence_available = false + predictions.each do |p| + confidence_available |= p.confidence_values_available? + end res = [] conf_column = nil predictions.each do |p| @@ -269,9 +269,9 @@ module Lib a << nil end end - if p.confidence_values_available? + if confidence_available conf_column = a.size if conf_column==nil - a << p.confidence_value(i) #(format ? p.confidence_value(i).to_nice_s : p.confidence_value(i)) + a << p.confidence_value(i) end a << p.identifier(i) res << a @@ -279,6 +279,7 @@ module Lib end if conf_column!=nil + LOGGER.debug "sort via confidence: "+res.collect{|n| n[conf_column]}.inspect res = res.sort_by{ |n| n[conf_column] || 0 }.reverse if format res.each do |a| diff --git a/lib/predictions.rb b/lib/predictions.rb index 4a15e1d..a449776 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -45,16 +45,6 @@ module Lib raise "illegal num confidence values "+num_info if @confidence_values.size != @predicted_values.size @confidence_values.each{ |c| raise "illegal confidence value: '"+c.to_s+"'" unless c==nil or (c.is_a?(Numeric) and c>=0 and c<=1) } - ## check if there is more than one different conf value - ## DEPRECATED? not sure anymore what this was about, - ## I am pretty sure this was for r-plot of roc curves - ## roc curvers are now plotted manually - #conf_val_tmp = {} - #@confidence_values.each{ |c| conf_val_tmp[c] = nil } - #if conf_val_tmp.keys.size<2 - # LOGGER.warn("prediction w/o confidence values"); - # @confidence_values=nil - #end case @feature_type when "classification" @@ -75,11 +65,13 @@ module Lib init_stats() (0..@predicted_values.size-1).each do |i| - update_stats( @predicted_values[i], @actual_values[i], (@confidence_values!=nil)?@confidence_values[i]:nil ) + update_stats( @predicted_values[i], @actual_values[i], @confidence_values[i] ) end end def init_stats + @conf_provided = false + @num_no_actual_value = 0 @num_with_actual_value = 0 @@ -134,6 +126,8 @@ module Lib else @num_predicted += 1 + @conf_provided |= confidence_value!=nil + case @feature_type when "classification" @confusion_matrix[actual_value][predicted_value] += 1 @@ -186,6 +180,7 @@ module Lib end def weighted_accuracy + return 0 unless confidence_values_available? raise "no classification" unless @feature_type=="classification" total = 0 correct = 0 @@ -255,7 +250,7 @@ module Lib def area_under_roc(class_index=nil) return prediction_feature_value_map( lambda{ |i| area_under_roc(i) } ) if class_index==nil - return 0.0 if @confidence_values==nil + return 0 unless confidence_values_available? LOGGER.warn("TODO: implement approx computiation of AUC,"+ "so far Wilcoxon-Man-Whitney is used (exponential)") if @@ -485,7 +480,9 @@ module Lib # see http://en.wikipedia.org/wiki/Coefficient_of_determination#Definitions # see http://web.maths.unsw.edu.au/~adelle/Garvan/Assays/GoodnessOfFit.html - r_2 = 1 - residual_sum_of_squares / total_sum_of_squares + ss_tot = total_sum_of_squares + return 0 if ss_tot==0 + r_2 = 1 - residual_sum_of_squares / ss_tot ( r_2.infinite? || r_2.nan? ) ? 0 : r_2 end @@ -523,7 +520,7 @@ module Lib def get_prediction_values(class_value) #puts "get_roc_values for class_value: "+class_value.to_s - raise "no confidence values" if @confidence_values==nil + raise "no confidence values" unless confidence_values_available? #raise "no class-value specified" if class_value==nil class_index = @accept_values.index(class_value) if class_value!=nil @@ -594,7 +591,7 @@ module Lib end def confidence_values_available? - return @confidence_values!=nil + @conf_provided end ################################################################################################################### diff --git a/report/report_content.rb b/report/report_content.rb index d60f700..755147d 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -287,12 +287,9 @@ class Reports::ReportContent section_bar = @xml_report.add_section(@current_section, section_title) @xml_report.add_paragraph(section_bar, section_text) if section_text - - plot_file_name = "bar_plot"+@tmp_file_count.to_s+".png" - @tmp_file_count += 1 - plot_file_path = add_tmp_file(plot_file_name) - Reports::PlotFactory.create_bar_plot(plot_file_path, validation_set, title_attribute, value_attributes ) - @xml_report.add_imagefigure(section_bar, image_title, plot_file_name, "PNG", 100, image_caption) + plot_bar = add_tmp_file("bar_plot", "svg") + Reports::PlotFactory.create_bar_plot(plot_bar[:path], validation_set, title_attribute, value_attributes ) + @xml_report.add_imagefigure(section_bar, image_title, plot_bar[:name], "PNG", 100, image_caption) end private diff --git a/report/validation_access.rb b/report/validation_access.rb index ffb7461..d421a0b 100755 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -100,15 +100,17 @@ class Reports::ValidationDB def get_predictions(validation, subjectid=nil, task=nil) Lib::OTPredictions.new( validation.feature_type, validation.test_dataset_uri, validation.test_target_dataset_uri, validation.prediction_feature, validation.prediction_dataset_uri, - validation.predicted_variable, subjectid, task) + validation.predicted_variable, validation.predicted_confidence, subjectid, task) end def get_accept_values( validation, subjectid=nil ) # PENDING So far, one has to load the whole dataset to get the accept_value from ambit - d = Lib::DatasetCache.find( validation.test_target_dataset_uri, subjectid ) - raise "cannot get test target dataset for accept values, dataset: "+validation.test_target_dataset_uri.to_s unless d + test_target_dataset = validation.test_target_dataset_uri + test_target_dataset = validation.test_dataset_uri unless test_target_dataset + d = Lib::DatasetCache.find( test_target_dataset, subjectid ) + raise "cannot get test target dataset for accept values, dataset: "+test_target_dataset.to_s unless d accept_values = d.features[validation.prediction_feature][OT.acceptValue] - raise "cannot get accept values from dataset "+validation.test_target_dataset_uri.to_s+" for feature "+ + raise "cannot get accept values from dataset "+test_target_dataset.to_s+" for feature "+ validation.prediction_feature+":\n"+d.features[validation.prediction_feature].to_yaml unless accept_values!=nil accept_values end @@ -122,10 +124,16 @@ class Reports::ValidationDB raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) model = OpenTox::Model::Generic.find(validation.model_uri, subjectid) raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model - model.metadata[OT.predictedVariables] - #get_model(validation).predictedVariables + Lib::FeatureUtil.predicted_variables(model, validation.prediction_dataset_uri, subjectid)[:predicted_variable] end + def predicted_confidence(validation, subjectid=nil) + raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) + model = OpenTox::Model::Generic.find(validation.model_uri, subjectid) + raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model + Lib::FeatureUtil.predicted_variables(model, validation.prediction_dataset_uri, subjectid)[:predicted_confidence] + end + # private # def get_model(validation) # raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) diff --git a/report/validation_data.rb b/report/validation_data.rb index 0c15dd2..2479977 100755 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -139,6 +139,11 @@ module Reports @predicted_variable = Reports.validation_access.predicted_variable(self, @subjectid) end + def predicted_confidence + return @predicted_confidence if @predicted_confidence!=nil + @predicted_confidence = Reports.validation_access.predicted_confidence(self, @subjectid) + end + # loads all crossvalidation attributes, of the corresponding cv into this object def load_cv_attributes raise "crossvalidation-id not set" unless @crossvalidation_id diff --git a/validation/validation_application.rb b/validation/validation_application.rb index d448c62..f64f74e 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -4,6 +4,7 @@ end require 'lib/dataset_cache.rb' +require 'lib/feature_util.rb' require 'validation/validation_service.rb' get '/crossvalidation/?' do @@ -443,21 +444,22 @@ post '/validate_datasets' do params[:validation_type] = "validate_datasets" if params[:model_uri] + raise OpenTox::BadRequestError.new "please specify 'model_uri' or set either 'classification' or 'regression' flag" if params[:classification] or params[:regression] v = Validation::Validation.create params v.subjectid = @subjectid v.compute_validation_stats_with_model(nil,false,task) else raise OpenTox::BadRequestError.new "please specify 'model_uri' or 'prediction_feature'" unless params[:prediction_feature] - raise OpenTox::BadRequestError.new "please specify 'model_uri' or 'predicted_feature'" unless params[:predicted_feature] + raise OpenTox::BadRequestError.new "please specify 'model_uri' or 'predicted_variable'" unless params[:predicted_variable] raise OpenTox::BadRequestError.new "please specify 'model_uri' or set either 'classification' or 'regression' flag" unless params[:classification] or params[:regression] - - predicted_feature = params.delete("predicted_feature") + predicted_variable = params.delete("predicted_variable") + predicted_confidence = params.delete("predicted_confidence") feature_type = "classification" if params.delete("classification")!=nil feature_type = "regression" if params.delete("regression")!=nil v = Validation::Validation.create params v.subjectid = @subjectid - v.compute_validation_stats(feature_type,predicted_feature,nil,nil,false,task) + v.compute_validation_stats(feature_type,predicted_variable,predicted_confidence,nil,nil,false,task) end v.validation_uri end diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 61f3a6e..73c15df 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -46,9 +46,15 @@ module Validation test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri} prediction_feature = vals.first.prediction_feature prediction_dataset_uris = vals.collect{|v| v.prediction_dataset_uri} - predicted_variables = models.collect{|m| m.metadata[OT.predictedVariables]} + predicted_variables = [] + predicted_confidences = [] + models.size.times do |i| + predicted = Lib::FeatureUtil.predicted_variables(models[i], prediction_dataset_uris[i], subjectid) + predicted_variables << predicted[:predicted_variable] + predicted_confidences << predicted[:predicted_confidence] + end prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, - prediction_dataset_uris, predicted_variables, subjectid ) + prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid ) v = Validation.new case feature_type @@ -218,13 +224,15 @@ module Validation dependentVariables = model.metadata[OT.dependentVariables] prediction_feature = self.prediction_feature ? nil : dependentVariables algorithm_uri = self.algorithm_uri ? nil : model.metadata[OT.algorithm] - predictedVariables = model.metadata[OT.predictedVariables] - compute_validation_stats( model.feature_type(self.subjectid), predictedVariables, + predicted_variables = Lib::FeatureUtil.predicted_variables(model, prediction_dataset_uri, subjectid) + predicted_variable = predicted_variables[:predicted_variable] + predicted_confidence = predicted_variables[:predicted_confidence] + compute_validation_stats( model.feature_type(self.subjectid), predicted_variable, predicted_confidence, prediction_feature, algorithm_uri, dry_run, task ) end - def compute_validation_stats( feature_type, predicted_feature, prediction_feature=nil, - algorithm_uri=nil, dry_run=false, task=nil ) + def compute_validation_stats( feature_type, predicted_variable, predicted_confidence, prediction_feature, + algorithm_uri, dry_run, task ) # self.attributes = { :prediction_feature => prediction_feature } if self.prediction_feature==nil && prediction_feature # self.attributes = { :algorithm_uri => algorithm_uri } if self.algorithm_uri==nil && algorithm_uri @@ -237,7 +245,7 @@ module Validation LOGGER.debug "computing prediction stats" prediction = Lib::OTPredictions.new( feature_type, self.test_dataset_uri, self.test_target_dataset_uri, self.prediction_feature, - self.prediction_dataset_uri, predicted_feature, self.subjectid, OpenTox::SubTask.create(task, 0, 80) ) + self.prediction_dataset_uri, predicted_variable, predicted_confidence, self.subjectid, OpenTox::SubTask.create(task, 0, 80) ) #reading datasets and computing the main stats is 80% the work unless dry_run -- cgit v1.2.3 From e97a2b12676bdb5670fc75199d12bd15b5dd17d5 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 24 May 2011 16:21:49 +0200 Subject: remove debug output --- lib/feature_util.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/feature_util.rb b/lib/feature_util.rb index 405d991..286c05e 100644 --- a/lib/feature_util.rb +++ b/lib/feature_util.rb @@ -42,19 +42,15 @@ module Lib d.load_features(subjectid) d.features.keys.each do |f| if d.features[f][OT.hasSource]==model.uri - puts "source matching" # PENDING identify confidence if f =~ /(?i)confidence/ - puts "conf matiching" raise "duplicate confidence feature, what to choose?" if predicted_confidence!=nil predicted_confidence = f elsif d.features[f][RDF.type].include? OT.ModelPrediction - puts "type include prediction" raise "duplicate predicted variable, what to choose?" if predicted_variable!=nil predicted_variable = f end end - puts d.features[f][OT.hasSource] end raise "could not estimate predicted variable" unless predicted_variable end -- cgit v1.2.3 From 9bafc6c362b0e595fcf78fde0d588937db9ae122 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 24 May 2011 17:29:07 +0200 Subject: add rdf version for crossvalidation statistics --- validation/validation_application.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index f64f74e..46c213d 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -122,6 +122,9 @@ get '/crossvalidation/:id/statistics' do "The averaged statistics for the crossvalidation." content_type "text/html" OpenTox.text_to_html v.to_yaml,@subjectid,related_links,description + when "application/rdf+xml" + content_type "application/rdf+xml" + v.to_rdf else content_type "application/x-yaml" v.to_yaml -- cgit v1.2.3 From 78564782ca749dd13f063f0a04070bef89377354 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 24 May 2011 19:44:06 +0200 Subject: adjust to ruby-plot version 0.5, simplification + support for various max youden-index points --- report/environment.rb | 2 +- report/plot_factory.rb | 101 +++++++++++++++++------------------------- test/test_examples.rb | 26 ++++++++++- validation/validation_test.rb | 32 ++++++++++--- 4 files changed, 92 insertions(+), 69 deletions(-) diff --git a/report/environment.rb b/report/environment.rb index bdb4a04..a7d454f 100755 --- a/report/environment.rb +++ b/report/environment.rb @@ -4,7 +4,7 @@ 'rexml/document', 'ruby-plot', 'opentox-ruby' ].each do |g| require g end -gem 'ruby-plot', "~>0.4.0" +gem 'ruby-plot', "~>0.5.0" #R.quit diff --git a/report/plot_factory.rb b/report/plot_factory.rb index dfb1369..d09e506 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -95,41 +95,28 @@ module Reports # * each of theses validation sets is plotted as a roc-curve # def self.create_roc_plot( out_files, validation_set, class_value, split_set_attribute=nil, - x_label="False positive rate", y_label="True Positive Rate", show_single_curves=false ) + x_label="False positive rate", y_label="True Positive Rate" ) out_files = [out_files] unless out_files.is_a?(Array) LOGGER.debug "creating roc plot for '"+validation_set.size.to_s+"' validations, out-files:"+out_files.inspect + data = [] if split_set_attribute attribute_values = validation_set.get_values(split_set_attribute) - names = [] - fp_rates = [] - tp_rates = [] attribute_values.each do |value| begin - data = transform_roc_predictions(validation_set.filter({split_set_attribute => value}), class_value, false) - names << value.to_s - fp_rates << data[:fp_rate][0] - tp_rates << data[:tp_rate][0] + data << transform_roc_predictions(validation_set.filter({split_set_attribute => value}), class_value, false ) rescue LOGGER.warn "could not create ROC plot for "+value.to_s end end - out_files.each do |out_file| - RubyPlot::plot_lines(out_file, "ROC-Plot", x_label, y_label, names, fp_rates, tp_rates ) - end else - data = transform_roc_predictions(validation_set, class_value, show_single_curves) - labels = [] - data[:youden].each do |points| - points.each do |point,confidence| - labels << ["confidence: "+confidence.to_nice_s, point[0], point[1]] - end - end - out_files.each do |out_file| - RubyPlot::plot_lines(out_file, "ROC-Plot", x_label, y_label, data[:names], data[:fp_rate], data[:tp_rate], data[:faint], labels ) - end + data << transform_roc_predictions(validation_set, class_value ) end + + out_files.each do |out_file| + RubyPlot::plot_lines(out_file, "ROC-Plot", x_label, y_label, data ) + end end @@ -291,44 +278,27 @@ module Reports end private - def self.transform_roc_predictions(validation_set, class_value, add_single_folds=false) - + def self.transform_roc_predictions(validation_set, class_value, add_label=true ) if (validation_set.size > 1) - - names = []; fp_rate = []; tp_rate = []; faint = []; youden = [] - sum_roc_values = { :predicted_values => [], :actual_values => [], :confidence_values => []} - + values = { :predicted_values => [], :actual_values => [], :confidence_values => []} (0..validation_set.size-1).each do |i| roc_values = validation_set.get(i).get_predictions.get_prediction_values(class_value) - sum_roc_values[:predicted_values] += roc_values[:predicted_values] - sum_roc_values[:confidence_values] += roc_values[:confidence_values] - sum_roc_values[:actual_values] += roc_values[:actual_values] - if add_single_folds - begin - tp_fp_rates = get_tp_fp_rates(roc_values) - names << "fold "+i.to_s - fp_rate << tp_fp_rates[:fp_rate] - tp_rate << tp_fp_rates[:tp_rate] - faint << true - rescue - LOGGER.warn "could not get ROC vals for fold "+i.to_s - end - end + values[:predicted_values] += roc_values[:predicted_values] + values[:confidence_values] += roc_values[:confidence_values] + values[:actual_values] += roc_values[:actual_values] end - tp_fp_rates = get_tp_fp_rates(sum_roc_values) - names << nil # "all" - fp_rate << tp_fp_rates[:fp_rate] - tp_rate << tp_fp_rates[:tp_rate] - youden << tp_fp_rates[:youden] - faint << false - return { :names => names, :fp_rate => fp_rate, :tp_rate => tp_rate, :faint => faint, :youden => youden } else - roc_values = validation_set.validations[0].get_predictions.get_prediction_values(class_value) - tp_fp_rates = get_tp_fp_rates(roc_values) - return { :names => ["default"], :fp_rate => [tp_fp_rates[:fp_rate]], :tp_rate => [tp_fp_rates[:tp_rate]], :youden => [tp_fp_rates[:youden]] } + values = validation_set.validations[0].get_predictions.get_prediction_values(class_value) end + tp_fp_rates = get_tp_fp_rates(values) + labels = [] + tp_fp_rates[:youden].each do |point,confidence| + labels << ["confidence: "+confidence.to_nice_s, point[0], point[1]] + end if add_label + RubyPlot::LinePlotData.new(:name => "default", :x_values => tp_fp_rates[:fp_rate], :y_values => tp_fp_rates[:tp_rate], :labels => labels) end + def self.transform_confidence_predictions(validation_set, class_value, add_single_folds=false) if (validation_set.size > 1) @@ -368,16 +338,25 @@ module Reports end end - def self.demo_rock_plot - roc_values = {:confidence_values => [0.1, 0.9, 0.5, 0.6, 0.6, 0.6], - :predicted_values => [1, 0, 0, 1, 0, 1], - :actual_values => [0, 1, 0, 0, 1, 1]} + def self.demo_roc_plot +# roc_values = {:confidence_values => [0.1, 0.9, 0.5, 0.6, 0.6, 0.6], +# :predicted_values => [1, 0, 0, 1, 0, 1], +# :actual_values => [0, 1, 0, 0, 1, 1]} + roc_values = {:confidence_values => [0.9, 0.8, 0.7, 0.6, 0.5, 0.4], + :predicted_values => [1, 1, 1, 1, 1, 1], + :actual_values => [1, 0, 1, 0, 1, 0]} tp_fp_rates = get_tp_fp_rates(roc_values) - data = { :names => ["default"], :fp_rate => [tp_fp_rates[:fp_rate]], :tp_rate => [tp_fp_rates[:tp_rate]] } + labels = [] + tp_fp_rates[:youden].each do |point,confidence| + labels << ["confidence: "+confidence.to_s, point[0], point[1]] + end + + plot_data = [] + plot_data << RubyPlot::LinePlotData.new(:name => "testname", :x_values => tp_fp_rates[:fp_rate], :y_values => tp_fp_rates[:tp_rate], :labels => labels) RubyPlot::plot_lines("/tmp/plot.png", "ROC-Plot", "False positive rate", - "True Positive Rate", data[:names], data[:fp_rate], data[:tp_rate], data[:faint] ) + "True Positive Rate", plot_data ) end def self.get_performance_confidence_rates(roc_values, feature_type) @@ -503,7 +482,9 @@ module Reports max = youden.max youden_hash = {} (0..tp_rate.size-1).each do |i| - youden_hash[i] = c2[i] if youden[i]==max + if youden[i]==max and i>0 + youden_hash[i] = c2[i] + end end #puts youden.inspect+"\n"+youden_hash.inspect+"\n\n" @@ -526,8 +507,8 @@ end #require "rubygems" #require "ruby-plot" -#Reports::PlotFactory::demo_ranking_plot -#Reports::PlotFactory::demo_rock_plot +##Reports::PlotFactory::demo_ranking_plot +#Reports::PlotFactory::demo_roc_plot #a = [1, 0, 1, 2, 3, 0, 2] #puts a.compress_sum([100, 90, 70, 70, 30, 10, 0]).inspect diff --git a/test/test_examples.rb b/test/test_examples.rb index eb0543f..f3c0b7e 100755 --- a/test/test_examples.rb +++ b/test/test_examples.rb @@ -100,7 +100,7 @@ module ValidationExamples class EPAFHMCrossvalidation < CrossValidation def initialize - @dataset_file = File.new("data/EPAFHM.med.csv","r") + @dataset_file = File.new("data/EPAFHM.csv","r") #@prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" @num_folds = 10 end @@ -505,6 +505,26 @@ module ValidationExamples end end + class AnotherAmbitJ48TrainingTest < TrainingTestValidation + def initialize + @algorithm_uri = "http://apps.ideaconsult.net:8080/ambit2/algorithm/J48" + @training_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/585758" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/585758" + @prediction_feature= "http://apps.ideaconsult.net:8080/ambit2/feature/111148" + end + end + + class TumTrainingTest < TrainingTestValidation + def initialize + @algorithm_uri = "http://lxkramer34.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNclassification" + @training_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/585758" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/585758" + @prediction_feature= "http://apps.ideaconsult.net:8080/ambit2/feature/111148" + end + end + + + class LazarVsNtuaCrossvalidation < CrossValidation def initialize @@ -803,7 +823,11 @@ module ValidationExamples "22b" => [ NtuaTrainingTestSplit ], "22c" => [ NtuaCrossvalidation ], "22d" => [ LazarVsNtuaCrossvalidation ], + + #impt "22e" => [ AmbitVsNtuaTrainingTest ], + "22f" => [ AnotherAmbitJ48TrainingTest ], + "22g" => [ TumTrainingTest ], } diff --git a/validation/validation_test.rb b/validation/validation_test.rb index 2c86548..ae71749 100755 --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -60,7 +60,23 @@ class ValidationTest < Test::Unit::TestCase begin $test_case = self - #get 'crossvalidation/138/statistics' +# post "/validate_datasets",{ +# :test_dataset_uri=>"http://local-ot/dataset/6907", +# :prediction_dataset_uri=>"http://local-ot/dataset/6909", +# :test_target_dataset_uri=>"http://local-ot/dataset/6905", +# :prediction_feature=>"http://local-ot/dataset/6905/feature/Hamster%20Carcinogenicity", +# #:model_uri=>"http://local-ot/model/1078", +# :predicted_variable=>"http://local-ot/dataset/6909/feature/prediction/Hamster%20Carcinogenicity/value", +# :predicted_confidence=>"http://local-ot/dataset/6909/feature/prediction/Hamster%20Carcinogenicity/confidence", +# #:regression=>"true"} +# :classification=>"true"} +# +# puts last_response.body +# uri = last_response.body +# rep = wait_for_task(uri) +# puts rep + + #get 'crossvalidation/19/statistics' #get 'crossvalidation/189/statistics' #puts last_response.body # run_test("1b") @@ -68,7 +84,9 @@ class ValidationTest < Test::Unit::TestCase #get '/crossvalidation/79/predictions',nil,'HTTP_ACCEPT' => "application/x-yaml" #puts last_response.body - #run_test("22e") #,:validation_uri=>"http://local-ot/validation/84" ) + run_test("22f") #,:validation_uri=>"http://local-ot/validation/84" ) + + #run_test("21b") #run_test("21c") @@ -97,16 +115,16 @@ class ValidationTest < Test::Unit::TestCase # rep = wait_for_task(uri) # puts rep - #run_test("1a", {:validation_uri=>"http://local-ot/validation/466"}) + #run_test("1a", {:validation_uri=>"http://local-ot/validation/305"}) # puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - #run_test("3a",{:validation_uri=>"http://local-ot/validation/crossvalidation/149"}) + #run_test("3a",{:validation_uri=>"http://local-ot/validation/crossvalidation/6"}) #puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - #run_test("13a", {:validation_uri=>"http://local-ot/validation/406"}) + #run_test("13a") #, {:validation_uri=>"http://local-ot/validation/406"}) # puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - #run_test("14a",{:validation_uri=>"http://local-ot/validation/crossvalidation/148"}) + #run_test("14a") #,{:validation_uri=>"http://local-ot/validation/crossvalidation/148"}) # puts "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - run_test("1a") + #run_test("1a") # run_test("3d",{ # :dataset_uri => "http://local-ot/dataset/2897", -- cgit v1.2.3 From bc97f210ab0e50344ea7fdc047c21fb74a24375f Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 25 May 2011 10:07:33 +0200 Subject: add cv debug msg to see number of folds --- validation/validation_service.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 73c15df..e16e45d 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -331,6 +331,7 @@ module Validation raise "validation '"+validation.validation_uri+"' for crossvaldation could not be finished" unless validation.finished i += 1 + LOGGER.debug "fold "+i.to_s+" done: "+validation.validation_uri.to_s end # self.attributes = { :finished => true } -- cgit v1.2.3 From f951dac72a5505fb41af38359977b8c78a951f76 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 25 May 2011 10:35:37 +0200 Subject: replace +- in reports with corresponding html-char --- report/xml_report.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/report/xml_report.rb b/report/xml_report.rb index 855a65e..c2dd300 100755 --- a/report/xml_report.rb +++ b/report/xml_report.rb @@ -181,7 +181,7 @@ module Reports # def add_table( element, title, table_values, first_row_header=true, first_col_header=false, transpose=false, auto_link_urls=true ) - raise "table_values is not mulit-dimensional-array" unless table_values && table_values.is_a?(Array) && table_values[0].is_a?(Array) + raise "table_values is not multi-dimensional-array" unless table_values && table_values.is_a?(Array) && table_values[0].is_a?(Array) values = transpose ? table_values.transpose : table_values @@ -231,8 +231,10 @@ module Reports space.text = " " entry << space end - else - entry.text = v.to_s + else + text = v.to_s + text.gsub!(/\+\-/,"±") + entry << Text.new(text, true, nil, true) end row << entry end -- cgit v1.2.3 From b71b7696f4e628cb6b50703375203a9ef16ea19a Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 25 May 2011 14:07:57 +0200 Subject: sort validation and crossvalidation lists --- validation/validation_application.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 46c213d..38fa1c7 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -9,7 +9,7 @@ require 'validation/validation_service.rb' get '/crossvalidation/?' do LOGGER.info "list all crossvalidations" - uri_list = Lib::OhmUtil.find( Validation::Crossvalidation, params ).collect{|v| v.crossvalidation_uri}.join("\n") + "\n" + uri_list = Lib::OhmUtil.find( Validation::Crossvalidation, params ).sort.collect{|v| v.crossvalidation_uri}.join("\n") + "\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "Single validations: "+url_for("/",:full)+"\n"+ @@ -190,7 +190,7 @@ end get '/?' do LOGGER.info "list all validations, params: "+params.inspect - uri_list = Lib::OhmUtil.find( Validation::Validation, params ).collect{|v| v.validation_uri}.join("\n") + "\n" + uri_list = Lib::OhmUtil.find( Validation::Validation, params ).sort.collect{|v| v.validation_uri}.join("\n") + "\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = "To perform a validation:\n"+ @@ -244,7 +244,7 @@ get '/test_set_validation' do #uri_list = Validation::Validation.all( :validation_type => "test_set_validation" ).collect{ |v| v.validation_uri }.join("\n")+"\n" #params[:validation_type] = "test_set_validation" #uri_list = Lib::DataMapperUtil.all(Validation::Validation,params).collect{ |v| v.validation_uri }.join("\n")+"\n" - uri_list = Validation::Validation.find(:validation_type => "test_set_validation").collect{|v| v.validation_uri}.join("\n") + "\n" + uri_list = Validation::Validation.find(:validation_type => "test_set_validation").sort.collect{|v| v.validation_uri}.join("\n") + "\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = @@ -289,7 +289,7 @@ get '/training_test_validation' do #uri_list = Validation::Validation.all( :validation_type => "training_test_validation" ).collect{ |v| v.validation_uri }.join("\n")+"\n" #params[:validation_type] = "training_test_validation" #uri_list = Lib::DataMapperUtil.all(Validation::Validation,params).collect{ |v| v.validation_uri }.join("\n")+"\n" - uri_list = Validation::Validation.find(:validation_type => "training_test_validation").collect{|v| v.validation_uri}.join("\n") + "\n" + uri_list = Validation::Validation.find(:validation_type => "training_test_validation").sort.collect{|v| v.validation_uri}.join("\n") + "\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = @@ -339,7 +339,7 @@ get '/bootstrapping' do #uri_list = Validation::Validation.all( :validation_type => "bootstrapping" ).collect{ |v| v.validation_uri }.join("\n")+"\n" #params[:validation_type] = "bootstrapping" #uri_list = Lib::DataMapperUtil.all(Validation::Validation,params).collect{ |v| v.validation_uri }.join("\n")+"\n" - uri_list = Validation::Validation.find(:validation_type => "bootstrapping").collect{|v| v.validation_uri}.join("\n") + "\n" + uri_list = Validation::Validation.find(:validation_type => "bootstrapping").sort.collect{|v| v.validation_uri}.join("\n") + "\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = @@ -391,7 +391,7 @@ get '/training_test_split' do #uri_list = Validation::Validation.all( :validation_type => "training_test_split" ).collect{ |v| v.validation_uri }.join("\n")+"\n" #params[:validation_type] = "training_test_split" #uri_list = Lib::DataMapperUtil.all(Validation::Validation,params).collect{ |v| v.validation_uri }.join("\n")+"\n" - uri_list = Validation::Validation.find(:validation_type => "training_test_split").collect{|v| v.validation_uri}.join("\n") + "\n" + uri_list = Validation::Validation.find(:validation_type => "training_test_split").sort.collect{|v| v.validation_uri}.join("\n") + "\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = -- cgit v1.2.3 From f694a575013fef53ef27a1e4415c4039531fbfbe Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 25 May 2011 14:08:58 +0200 Subject: fix: bar-plot format changed from svg to png --- report/plot_factory.rb | 9 ++++++--- report/report_content.rb | 7 ++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/report/plot_factory.rb b/report/plot_factory.rb index d09e506..78d2e05 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -163,9 +163,10 @@ module Reports end - def self.create_bar_plot( out_file, validation_set, title_attribute, value_attributes ) + def self.create_bar_plot( out_files, validation_set, title_attribute, value_attributes ) - LOGGER.debug "creating bar plot, out-file:"+out_file.to_s + out_files = [out_files] unless out_files.is_a?(Array) + LOGGER.debug "creating bar plot, out-files:"+out_files.inspect data = [] titles = [] @@ -214,7 +215,9 @@ module Reports LOGGER.debug "bar plot labels: "+labels.inspect LOGGER.debug "bar plot data: "+data.inspect - RubyPlot::plot_bars('Bar plot', labels, data, out_file) + out_files.each do |out_file| + RubyPlot::plot_bars('Bar plot', labels, data, out_file) + end end diff --git a/report/report_content.rb b/report/report_content.rb index 755147d..a554881 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -287,9 +287,10 @@ class Reports::ReportContent section_bar = @xml_report.add_section(@current_section, section_title) @xml_report.add_paragraph(section_bar, section_text) if section_text - plot_bar = add_tmp_file("bar_plot", "svg") - Reports::PlotFactory.create_bar_plot(plot_bar[:path], validation_set, title_attribute, value_attributes ) - @xml_report.add_imagefigure(section_bar, image_title, plot_bar[:name], "PNG", 100, image_caption) + plot_png = add_tmp_file("bar_plot", "png") + plot_svg = add_tmp_file("bar_plot", "svg") + Reports::PlotFactory.create_bar_plot([plot_png[:path], plot_svg[:path]], validation_set, title_attribute, value_attributes ) + @xml_report.add_imagefigure(section_bar, image_title, plot_png[:name], "PNG", 100, plot_svg[:name]) end private -- cgit v1.2.3 From 8b8d7bc91ee841461620269f5892da72a0b114f6 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 26 May 2011 10:57:32 +0200 Subject: use accept_values method instead of accessing metadata directly --- lib/ot_predictions.rb | 2 +- report/validation_access.rb | 2 +- validation/validation_service.rb | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index df1efc3..d0530a3 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -81,7 +81,7 @@ module Lib raise "test dataset is empty "+test_dataset_uri.to_s unless compounds.size>0 if feature_type=="classification" - av = test_target_dataset.features[prediction_feature][OT.acceptValue] + av = test_target_dataset.accept_values(prediction_feature) raise "'"+OT.acceptValue.to_s+"' missing/invalid for feature '"+prediction_feature.to_s+"' in dataset '"+ test_target_dataset_uri.to_s+"', acceptValues are: '"+av.inspect+"'" if av==nil or av.length<2 if accept_values==nil diff --git a/report/validation_access.rb b/report/validation_access.rb index d421a0b..ef6a806 100755 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -109,7 +109,7 @@ class Reports::ValidationDB test_target_dataset = validation.test_dataset_uri unless test_target_dataset d = Lib::DatasetCache.find( test_target_dataset, subjectid ) raise "cannot get test target dataset for accept values, dataset: "+test_target_dataset.to_s unless d - accept_values = d.features[validation.prediction_feature][OT.acceptValue] + accept_values = d.accept_values(validation.prediction_feature) raise "cannot get accept values from dataset "+test_target_dataset.to_s+" for feature "+ validation.prediction_feature+":\n"+d.features[validation.prediction_feature].to_yaml unless accept_values!=nil accept_values diff --git a/validation/validation_service.rb b/validation/validation_service.rb index e16e45d..6954a33 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -392,7 +392,9 @@ module Validation split_compounds = shuffled_compounds.chunk( self.num_folds.to_i ) else class_compounds = {} # "inactive" => compounds[], "active" => compounds[] .. - accept_values = orig_dataset.features[prediction_feature][OT.acceptValue] + accept_values = orig_dataset.accept_values(prediction_feature) + raise BadRequestError.new("cannot apply stratification (not implemented for regression), acceptValue missing for prediction-feature '"+ + prediction_feature.to_s+"' in dataset '"+dataset_uri.to_s+"'") unless accept_values and accept_values.size>0 accept_values.each do |value| class_compounds[value] = [] shuffled_compounds.each do |c| -- cgit v1.2.3 From 4024dd30788865f019b2d2d2ed705a4945e8c52e Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 26 May 2011 12:07:33 +0200 Subject: fix error for stratified regression cv --- validation/validation_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 6954a33..d135a2c 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -393,7 +393,7 @@ module Validation else class_compounds = {} # "inactive" => compounds[], "active" => compounds[] .. accept_values = orig_dataset.accept_values(prediction_feature) - raise BadRequestError.new("cannot apply stratification (not implemented for regression), acceptValue missing for prediction-feature '"+ + raise OpenTox::BadRequestError.new("cannot apply stratification (not implemented for regression), acceptValue missing for prediction-feature '"+ prediction_feature.to_s+"' in dataset '"+dataset_uri.to_s+"'") unless accept_values and accept_values.size>0 accept_values.each do |value| class_compounds[value] = [] -- cgit v1.2.3 From 0d765f29d4d98cb085d8c067acf4d2249b541c6e Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 26 May 2011 14:15:54 +0200 Subject: fixes in reports for crossvaldiation with few predictions --- report/report_content.rb | 6 ++---- report/xml_report.rb | 16 +++++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/report/report_content.rb b/report/report_content.rb index a554881..d45d736 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -164,8 +164,7 @@ class Reports::ReportContent accept_value, split_set_attribute=nil, image_title = "ROC Plot", - section_text=nil, - image_caption=nil) + section_text="") #section_roc = @xml_report.add_section(@current_section, section_title) section_roc = @current_section @@ -282,8 +281,7 @@ class Reports::ReportContent value_attributes, section_title="Bar Plot", section_text=nil, - image_title="Bar Plot", - image_caption=nil) + image_title="Bar Plot") section_bar = @xml_report.add_section(@current_section, section_title) @xml_report.add_paragraph(section_bar, section_text) if section_text diff --git a/report/xml_report.rb b/report/xml_report.rb index c2dd300..ab3098e 100755 --- a/report/xml_report.rb +++ b/report/xml_report.rb @@ -133,16 +133,18 @@ module Reports return figure end - # bit of a hack to algin the last two elements that have been added to element into one row - def align_last_two_elements( element, title ) + # bit of a hack to algin the last two figures that have been added to element into one row + def align_last_two_figures( element, title ) imgs = [] - element.elements.each do |e| + element.elements.each do |e| imgs[0] = imgs[1] - imgs[1] = e + imgs[1] = e if e.name=="figure" + end + if (imgs[0] and imgs[1]) + element.delete_element imgs[0] + element.delete_element imgs[1] + add_imagefigures_in_row( element, imgs, title ) end - element.delete_element imgs[0] - element.delete_element imgs[1] - add_imagefigures_in_row( element, imgs, title ) end def add_imagefigures_in_row( element, imagefigures, title ) -- cgit v1.2.3 From 72c7b6bd957afe6df71e1f4afe08c7b1f1669f09 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 26 May 2011 15:16:26 +0200 Subject: fix for last commit: fixes in reports for crossvaldiation with few predictions --- report/report_content.rb | 2 +- report/xml_report.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/report/report_content.rb b/report/report_content.rb index d45d736..d151a7d 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -126,7 +126,7 @@ class Reports::ReportContent # bit of a hack to algin the last two plots in the report in to one row def align_last_two_images( title ) - @xml_report.align_last_two_elements(@current_section, title ) + @xml_report.align_last_two_images(@current_section, title ) end def add_regression_plot( validation_set, diff --git a/report/xml_report.rb b/report/xml_report.rb index ab3098e..5be5fdc 100755 --- a/report/xml_report.rb +++ b/report/xml_report.rb @@ -134,7 +134,7 @@ module Reports end # bit of a hack to algin the last two figures that have been added to element into one row - def align_last_two_figures( element, title ) + def align_last_two_images( element, title ) imgs = [] element.elements.each do |e| imgs[0] = imgs[1] -- cgit v1.2.3 From c70021d0ce1d7bb68a609f24f9888fc2285fe774 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 26 May 2011 15:18:02 +0200 Subject: A&A reporting fix --- report/validation_data.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/report/validation_data.rb b/report/validation_data.rb index 2479977..fa0af8e 100755 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -85,7 +85,7 @@ module Reports VAL_ATTR_RANKING.collect{ |a| (a.to_s+"_ranking").to_sym } @@validation_attributes.each{ |a| attr_accessor a } - attr_reader :predictions + attr_reader :predictions, :subjectid attr_accessor :identifier, :validation_report_uri, :crossvalidation_report_uri def initialize(uri = nil, subjectid = nil) @@ -412,7 +412,7 @@ module Reports new_set = ValidationSet.new grouping = Util.group(@validations, [:crossvalidation_id]) grouping.each do |g| - v = ReportValidation.from_cv_statistics(g[0].crossvalidation_uri) + v = ReportValidation.from_cv_statistics(g[0].crossvalidation_uri, g[0].subjectid) v.identifier = g.collect{|vv| vv.identifier}.uniq.join(";") new_set.validations << v end -- cgit v1.2.3 From 5abed3eedb5e0bc763574ef6a24b006546bdb22f Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 26 May 2011 18:06:50 +0200 Subject: implemented loo crossvalidation --- lib/feature_util.rb | 3 ++- lib/validation_db.rb | 2 ++ validation/validation_application.rb | 48 ++++++++++++++++++++++++++++++++---- validation/validation_service.rb | 20 ++++++++++++--- 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/lib/feature_util.rb b/lib/feature_util.rb index 286c05e..e412416 100644 --- a/lib/feature_util.rb +++ b/lib/feature_util.rb @@ -52,7 +52,8 @@ module Lib end end end - raise "could not estimate predicted variable" unless predicted_variable + raise "could not estimate predicted variable, model: '"+model.uri.to_s+"', prediction_dataset: '"+ + prediction_dataset_uri.to_s+"'" unless predicted_variable end {:predicted_variable => predicted_variable, :predicted_confidence => predicted_confidence} diff --git a/lib/validation_db.rb b/lib/validation_db.rb index e2595c5..c38b82e 100755 --- a/lib/validation_db.rb +++ b/lib/validation_db.rb @@ -136,6 +136,7 @@ module Validation attribute :random_seed attribute :finished attribute :stratified + attribute :loo attr_accessor :subjectid @@ -145,6 +146,7 @@ module Validation index :random_seed index :stratified index :finished + index :loo def self.create(params={}) params[:date] = Time.new diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 38fa1c7..32a1c99 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -12,8 +12,9 @@ get '/crossvalidation/?' do uri_list = Lib::OhmUtil.find( Validation::Crossvalidation, params ).sort.collect{|v| v.crossvalidation_uri}.join("\n") + "\n" if request.env['HTTP_ACCEPT'] =~ /text\/html/ related_links = - "Single validations: "+url_for("/",:full)+"\n"+ - "Crossvalidation reports: "+url_for("/report/crossvalidation",:full) + "Single validations: "+url_for("/",:full)+"\n"+ + "Leave-one-out crossvalidations: "+url_for("/crossvalidation/loo",:full)+"\n"+ + "Crossvalidation reports: "+url_for("/report/crossvalidation",:full) description = "A list of all crossvalidations.\n"+ "Use the POST method to perform a crossvalidation." @@ -36,7 +37,8 @@ post '/crossvalidation/?' do params[:num_folds].to_i>1 cv_params = { :dataset_uri => params[:dataset_uri], - :algorithm_uri => params[:algorithm_uri] } + :algorithm_uri => params[:algorithm_uri], + :loo => "false" } [ :num_folds, :random_seed ].each{ |sym| cv_params[sym] = params[sym] if params[sym] } cv_params[:stratified] = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] cv = Validation::Crossvalidation.create cv_params @@ -70,11 +72,47 @@ post '/crossvalidation/cleanup/?' do end post '/crossvalidation/loo/?' do - raise "not yet implemented" + task = OpenTox::Task.create( "Perform loo-crossvalidation", url_for("/crossvalidation/loo", :full) ) do |task| #, params + LOGGER.info "creating loo-crossvalidation "+params.inspect + raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] + raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri] + raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature] + raise OpenTox::BadRequestError.new "illegal param: num_folds, stratified, random_seed not allowed for loo-crossvalidation" if params[:num_folds] or + params[:stratifed] or params[:random_seed] + + cv_params = { :dataset_uri => params[:dataset_uri], + :algorithm_uri => params[:algorithm_uri], + :loo => "true" } + cv = Validation::Crossvalidation.create cv_params + cv.subjectid = @subjectid + cv.perform_cv( params[:prediction_feature], params[:algorithm_params], task ) + # computation of stats is cheap as dataset are already loaded into the memory + Validation::Validation.from_cv_statistics( cv.id, @subjectid ) + cv.crossvalidation_uri + end + return_task(task) end get '/crossvalidation/loo/?' do - raise OpenTox::BadRequestError.new "GET operation not supported, use POST for performing a loo-crossvalidation, see "+url_for("/crossvalidation", :full)+" for crossvalidation results" + LOGGER.info "list all crossvalidations" + params[:loo]="true" + uri_list = Lib::OhmUtil.find( Validation::Crossvalidation, params ).sort.collect{|v| v.crossvalidation_uri}.join("\n") + "\n" + if request.env['HTTP_ACCEPT'] =~ /text\/html/ + related_links = + "Single validations: "+url_for("/",:full)+"\n"+ + "All crossvalidations: "+url_for("/crossvalidation",:full)+"\n"+ + "Crossvalidation reports: "+url_for("/report/crossvalidation",:full) + description = + "A list of all leave one out crossvalidations.\n"+ + "Use the POST method to perform a crossvalidation." + post_params = [[:dataset_uri,:algorithm_uri,:prediction_feature,[:algorithm_params,""]]] + content_type "text/html" + OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_params + else + content_type "text/uri-list" + uri_list + end + end get '/crossvalidation/:id' do diff --git a/validation/validation_service.rb b/validation/validation_service.rb index d135a2c..a18c6ab 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -306,9 +306,16 @@ module Validation # creates the cv folds def create_cv_datasets( prediction_feature, task=nil ) - self.random_seed = 1 unless self.random_seed - self.num_folds = 10 unless self.num_folds - self.stratified = false unless self.stratified + if self.loo=="true" + orig_dataset = Lib::DatasetCache.find(self.dataset_uri,self.subjectid) + self.num_folds = orig_dataset.compounds.size + self.random_seed = 0 + self.stratified = false + else + self.random_seed = 1 unless self.random_seed + self.num_folds = 10 unless self.num_folds + self.stratified = false unless self.stratified + end if copy_cv_datasets( prediction_feature ) # dataset folds of a previous crossvalidaiton could be used task.progress(100) if task @@ -350,6 +357,7 @@ module Validation :num_folds => self.num_folds, :stratified => self.stratified, :random_seed => self.random_seed, + :loo => self.loo, :finished => true} ).reject{ |cv| cv.id == self.id } cvs.each do |cv| next if AA_SERVER and !OpenTox::Authorization.authorized?(cv.crossvalidation_uri,"GET",self.subjectid) @@ -386,7 +394,11 @@ module Validation orig_dataset = Lib::DatasetCache.find(self.dataset_uri,self.subjectid) raise OpenTox::NotFoundError.new "Dataset not found: "+self.dataset_uri.to_s unless orig_dataset - shuffled_compounds = orig_dataset.compounds.shuffle( self.random_seed ) + if self.loo=="true" + shuffled_compounds = orig_dataset.compounds + else + shuffled_compounds = orig_dataset.compounds.shuffle( self.random_seed ) + end unless self.stratified split_compounds = shuffled_compounds.chunk( self.num_folds.to_i ) -- cgit v1.2.3 From 654711ccecf3829f9ba8501fe67df06fd1aac9cf Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 27 May 2011 10:09:35 +0200 Subject: remove hack to determine prediction feature --- lib/feature_util.rb | 63 ------------------------------------ lib/predictions.rb | 2 +- report/validation_access.rb | 6 ++-- validation/validation_application.rb | 1 - validation/validation_service.rb | 14 +++----- 5 files changed, 8 insertions(+), 78 deletions(-) delete mode 100644 lib/feature_util.rb diff --git a/lib/feature_util.rb b/lib/feature_util.rb deleted file mode 100644 index e412416..0000000 --- a/lib/feature_util.rb +++ /dev/null @@ -1,63 +0,0 @@ - -module Lib - class FeatureUtil - - # this derieves the predicted_variable and predicted_confidence in prediction dataset - # predicted_variable: the feature-uri of model predictions - # predicted_confidence: the feature-uri of the model prediction confidence - # according to API info should be available in the API - # problem: IST has no feature-service -> predicted_variable depends on prediction dataset - # - # PENDING: ambit and ist declare prediction features differently -> unify - # - def self.predicted_variables( model, prediction_dataset_uri, subjectid=nil ) - - predicted_variable = nil - predicted_confidence = nil - - if model.metadata[OT.predictedVariables] - predictedVariables = model.metadata[OT.predictedVariables] - if predictedVariables.is_a?(Array) - if (predictedVariables.size==1) - predicted_variable = predictedVariables[0] - elsif (predictedVariables.size==2) - # PENDING identify confidence - conf_index = -1 - predictedVariables.size.times do |i| - conf_index = i if OpenTox::Feature.find(predictedVariables[i]).metadata[DC.title]=~/(?i)confidence/ - end - raise "size=2, no confidence "+predictedVariables.inspect+" "+model.uri.to_s if conf_index==-1 - predicted_variable = predictedVariables[1-conf_index] - predicted_confidence = predictedVariables[conf_index] - else - raise "size>2 "+predictedVariables.inspect+" "+model.uri.to_s - end - else - raise "predictedVariables is no array" - end - end - - unless predicted_variable - d = OpenTox::Dataset.new prediction_dataset_uri - d.load_features(subjectid) - d.features.keys.each do |f| - if d.features[f][OT.hasSource]==model.uri - # PENDING identify confidence - if f =~ /(?i)confidence/ - raise "duplicate confidence feature, what to choose?" if predicted_confidence!=nil - predicted_confidence = f - elsif d.features[f][RDF.type].include? OT.ModelPrediction - raise "duplicate predicted variable, what to choose?" if predicted_variable!=nil - predicted_variable = f - end - end - end - raise "could not estimate predicted variable, model: '"+model.uri.to_s+"', prediction_dataset: '"+ - prediction_dataset_uri.to_s+"'" unless predicted_variable - end - - {:predicted_variable => predicted_variable, :predicted_confidence => predicted_confidence} - end - end -end - diff --git a/lib/predictions.rb b/lib/predictions.rb index a449776..2409375 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -36,7 +36,7 @@ module Lib #puts "actual: "+actual_values.inspect #puts "confidence: "+confidence_values.inspect - raise "unknown feature_type: "+@feature_type.to_s unless + raise "unknown feature_type: '"+@feature_type.to_s+"'" unless @feature_type=="classification" || @feature_type=="regression" raise "no predictions" if @predicted_values.size == 0 num_info = "predicted:"+@predicted_values.size.to_s+ diff --git a/report/validation_access.rb b/report/validation_access.rb index ef6a806..299b124 100755 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -124,15 +124,15 @@ class Reports::ValidationDB raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) model = OpenTox::Model::Generic.find(validation.model_uri, subjectid) raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model - Lib::FeatureUtil.predicted_variables(model, validation.prediction_dataset_uri, subjectid)[:predicted_variable] + model.predicted_variable(subjectid) end def predicted_confidence(validation, subjectid=nil) raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) model = OpenTox::Model::Generic.find(validation.model_uri, subjectid) raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model - Lib::FeatureUtil.predicted_variables(model, validation.prediction_dataset_uri, subjectid)[:predicted_confidence] - end + model.predicted_confidence(subjectid) + end # private # def get_model(validation) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 32a1c99..39a5c50 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -4,7 +4,6 @@ end require 'lib/dataset_cache.rb' -require 'lib/feature_util.rb' require 'validation/validation_service.rb' get '/crossvalidation/?' do diff --git a/validation/validation_service.rb b/validation/validation_service.rb index a18c6ab..cc9faf5 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -46,13 +46,8 @@ module Validation test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri} prediction_feature = vals.first.prediction_feature prediction_dataset_uris = vals.collect{|v| v.prediction_dataset_uri} - predicted_variables = [] - predicted_confidences = [] - models.size.times do |i| - predicted = Lib::FeatureUtil.predicted_variables(models[i], prediction_dataset_uris[i], subjectid) - predicted_variables << predicted[:predicted_variable] - predicted_confidences << predicted[:predicted_confidence] - end + predicted_variables = models.collect{|m| m.predicted_variable(subjectid)} + predicted_confidences = models.collect{|m| m.predicted_confidence(subjectid)} prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid ) @@ -224,9 +219,8 @@ module Validation dependentVariables = model.metadata[OT.dependentVariables] prediction_feature = self.prediction_feature ? nil : dependentVariables algorithm_uri = self.algorithm_uri ? nil : model.metadata[OT.algorithm] - predicted_variables = Lib::FeatureUtil.predicted_variables(model, prediction_dataset_uri, subjectid) - predicted_variable = predicted_variables[:predicted_variable] - predicted_confidence = predicted_variables[:predicted_confidence] + predicted_variable = model.predicted_variable(self.subjectid) + predicted_confidence = model.predicted_confidence(self.subjectid) compute_validation_stats( model.feature_type(self.subjectid), predicted_variable, predicted_confidence, prediction_feature, algorithm_uri, dry_run, task ) end -- cgit v1.2.3 From 0b01ead59f4b07d81c10ce23a062272d7fdf0fe2 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 27 May 2011 10:38:53 +0200 Subject: adding more verbose error message when feature_type is unknown --- validation/validation_service.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index cc9faf5..5756913 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -216,12 +216,16 @@ module Validation model = OpenTox::Model::Generic.find(self.model_uri, self.subjectid) if model==nil and self.model_uri raise OpenTox::NotFoundError.new "model not found: "+self.model_uri.to_s unless model + feature_type = model.feature_type(self.subjectid) dependentVariables = model.metadata[OT.dependentVariables] prediction_feature = self.prediction_feature ? nil : dependentVariables algorithm_uri = self.algorithm_uri ? nil : model.metadata[OT.algorithm] predicted_variable = model.predicted_variable(self.subjectid) predicted_confidence = model.predicted_confidence(self.subjectid) - compute_validation_stats( model.feature_type(self.subjectid), predicted_variable, predicted_confidence, + raise "cannot determine whether model '"+model.uri.to-s+"' performs classification or regression, "+ + "please set rdf-type of predictedVariables feature '"+predicted_variable.to_s+ + "' to NominalFeature or NumericFeature" if (feature_type.to_s!="classification" and feature_type.to_s!="regression") + compute_validation_stats( feature_type, predicted_variable, predicted_confidence, prediction_feature, algorithm_uri, dry_run, task ) end -- cgit v1.2.3 From 2a2cc158359b29d290c511b5636ff00480b59475 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 27 May 2011 11:41:54 +0200 Subject: fix error msg --- validation/validation_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 5756913..0651622 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -222,7 +222,7 @@ module Validation algorithm_uri = self.algorithm_uri ? nil : model.metadata[OT.algorithm] predicted_variable = model.predicted_variable(self.subjectid) predicted_confidence = model.predicted_confidence(self.subjectid) - raise "cannot determine whether model '"+model.uri.to-s+"' performs classification or regression, "+ + raise "cannot determine whether model '"+model.uri.to_s+"' performs classification or regression, "+ "please set rdf-type of predictedVariables feature '"+predicted_variable.to_s+ "' to NominalFeature or NumericFeature" if (feature_type.to_s!="classification" and feature_type.to_s!="regression") compute_validation_stats( feature_type, predicted_variable, predicted_confidence, -- cgit v1.2.3 From 092eca3470f13d6b1df4afc5ea98a0d398300a37 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 27 May 2011 17:36:11 +0200 Subject: fix percentage completed for cv --- validation/validation_application.rb | 8 ++++---- validation/validation_service.rb | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 39a5c50..5ae6bda 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -42,9 +42,9 @@ post '/crossvalidation/?' do cv_params[:stratified] = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] cv = Validation::Crossvalidation.create cv_params cv.subjectid = @subjectid - cv.perform_cv( params[:prediction_feature], params[:algorithm_params], task ) + cv.perform_cv( params[:prediction_feature], params[:algorithm_params], OpenTox::SubTask.create(task,0,95)) # computation of stats is cheap as dataset are already loaded into the memory - Validation::Validation.from_cv_statistics( cv.id, @subjectid ) + Validation::Validation.from_cv_statistics( cv.id, @subjectid, OpenTox::SubTask.create(task,95,100) ) cv.crossvalidation_uri end return_task(task) @@ -84,9 +84,9 @@ post '/crossvalidation/loo/?' do :loo => "true" } cv = Validation::Crossvalidation.create cv_params cv.subjectid = @subjectid - cv.perform_cv( params[:prediction_feature], params[:algorithm_params], task ) + cv.perform_cv( params[:prediction_feature], params[:algorithm_params], OpenTox::SubTask.create(task,0,95)) # computation of stats is cheap as dataset are already loaded into the memory - Validation::Validation.from_cv_statistics( cv.id, @subjectid ) + Validation::Validation.from_cv_statistics( cv.id, @subjectid, OpenTox::SubTask.create(task,95,100) ) cv.crossvalidation_uri end return_task(task) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 0651622..57e6f79 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -32,7 +32,7 @@ module Validation class Validation - def self.from_cv_statistics( cv_id, subjectid=nil ) + def self.from_cv_statistics( cv_id, subjectid=nil, waiting_task=nil ) v = Validation.find( :crossvalidation_id => cv_id, :validation_type => "crossvalidation_statistics" ).first unless v crossvalidation = Crossvalidation.get(cv_id) @@ -49,7 +49,7 @@ module Validation predicted_variables = models.collect{|m| m.predicted_variable(subjectid)} predicted_confidences = models.collect{|m| m.predicted_confidence(subjectid)} prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, - prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid ) + prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid, OpenTox::SubTask.create(waiting_task, 0, 90) ) v = Validation.new case feature_type @@ -74,6 +74,7 @@ module Validation v.real_runtime = vals.collect{ |vv| vv.real_runtime }.uniq.join(";") v.save end + waiting_task.progress(100) if waiting_task v end -- cgit v1.2.3 From f9f687ae583b8fdef2de7651e08b922ced3c0f37 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 30 May 2011 14:29:21 +0200 Subject: change qmrfreport address from qmrfreport2 to qmrfreport --- reach_reports/reach_application.rb | 102 ++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/reach_reports/reach_application.rb b/reach_reports/reach_application.rb index cd0695c..c3d6fbd 100755 --- a/reach_reports/reach_application.rb +++ b/reach_reports/reach_application.rb @@ -3,7 +3,7 @@ require lib end -QMRF_EDITOR_URI = "http://ortona.informatik.uni-freiburg.de/qmrfedit2/OT_QMRFEditor.jnlp" +QMRF_EDITOR_URI = "http://ortona.informatik.uni-freiburg.de/qmrfedit/OT_QMRFEditor.jnlp" require 'reach_reports/reach_persistance.rb' require 'reach_reports/reach_service.rb' @@ -151,63 +151,63 @@ get '/reach_report/:type/:id/editor' do jnlp = < - + QMRF Editor www.opentox.org (Q)SAR Model Reporting Format Editor (Q)SAR Model Reporting Format Editor - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -226,8 +226,8 @@ EOF jnlp += < --d http://opentox.informatik.uni-freiburg.de/qmrfedit2/qmrf.dtd --t http://opentox.informatik.uni-freiburg.de/qmrfedit2/verdana.ttf +-d http://opentox.informatik.uni-freiburg.de/qmrfedit/qmrf.dtd +-t http://opentox.informatik.uni-freiburg.de/qmrfedit/verdana.ttf -- cgit v1.2.3 From 9bec29d07a797090982e831a774c3aae532a9e9e Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 30 May 2011 15:02:38 +0200 Subject: fix algparams with multiple '=' --- validation/validation_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 57e6f79..8d64cd1 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -129,7 +129,7 @@ module Validation params = { :dataset_uri => self.training_dataset_uri, :prediction_feature => self.prediction_feature } if (algorithm_params!=nil) algorithm_params.split(";").each do |alg_params| - alg_param = alg_params.split("=") + alg_param = alg_params.split("=",2) raise OpenTox::BadRequestError.new "invalid algorithm param: '"+alg_params.to_s+"'" unless alg_param.size==2 or alg_param[0].to_s.size<1 or alg_param[1].to_s.size<1 LOGGER.warn "algorihtm param contains empty space, encode? "+alg_param[1].to_s if alg_param[1] =~ /\s/ params[alg_param[0].to_sym] = alg_param[1] -- cgit v1.2.3 From 43e211e2765ea3e929d6e146a11cdd8d5f87df30 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 31 May 2011 20:17:01 +0200 Subject: retry hack for mysql lite --- reach_reports/reach_application.rb | 14 ++++++++++++++ reach_reports/reach_persistance.rb | 15 +++++++++------ reach_reports/reach_service.rb | 21 ++++++++++++--------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/reach_reports/reach_application.rb b/reach_reports/reach_application.rb index c3d6fbd..d914452 100755 --- a/reach_reports/reach_application.rb +++ b/reach_reports/reach_application.rb @@ -5,6 +5,20 @@ end QMRF_EDITOR_URI = "http://ortona.informatik.uni-freiburg.de/qmrfedit/OT_QMRFEditor.jnlp" +# hack for as long as mysql lite is used +def mysql_lite_retry( n_times=15 ) + n_times.times do + begin + yield + return + rescue => ex + LOGGER.warn "datamapper error, wait and retry : "+ex.message + sleep(1+rand(3)) # wait 1-3 seconds + end + end + yield # try a last time +end + require 'reach_reports/reach_persistance.rb' require 'reach_reports/reach_service.rb' diff --git a/reach_reports/reach_persistance.rb b/reach_reports/reach_persistance.rb index b90778b..147ee84 100755 --- a/reach_reports/reach_persistance.rb +++ b/reach_reports/reach_persistance.rb @@ -1202,11 +1202,14 @@ module ReachReports # end # end - [ QsarSoftware, QsarIdentifier, QmrfAuthor, ModelAuthor, Reference, QsarGeneralInformation, ModelEndpoint, QsarEndpoint, AlgorithmExplicit, - AlgorithmsDescriptor, DescriptorsGenerationSoftware, QsarAlgorithm, AppDomainSoftware, QsarApplicabilityDomain, TrainingSetData, - QsarRobustness, ValidationSetData, QsarPredictivity, QsarInterpretation, Bibliography, AttachmentTrainingData, AttachmentValidationData, - AttachmentDocument, QsarMiscellaneous, QmrfSummary, QmrfReport ].each do |model| - model.auto_upgrade! - model.raise_on_save_failure = true + mysql_lite_retry do + [ QsarSoftware, QsarIdentifier, QmrfAuthor, ModelAuthor, Reference, QsarGeneralInformation, ModelEndpoint, QsarEndpoint, AlgorithmExplicit, + AlgorithmsDescriptor, DescriptorsGenerationSoftware, QsarAlgorithm, AppDomainSoftware, QsarApplicabilityDomain, TrainingSetData, + QsarRobustness, ValidationSetData, QsarPredictivity, QsarInterpretation, Bibliography, AttachmentTrainingData, AttachmentValidationData, + AttachmentDocument, QsarMiscellaneous, QmrfSummary, QmrfReport ].each do |model| + model.auto_upgrade! + model.raise_on_save_failure = true + end end + end \ No newline at end of file diff --git a/reach_reports/reach_service.rb b/reach_reports/reach_service.rb index b6c6350..916a7a4 100755 --- a/reach_reports/reach_service.rb +++ b/reach_reports/reach_service.rb @@ -279,8 +279,9 @@ module ReachReports end end task.progress(90) if task - - r.save + mysql_lite_retry do + r.save + end task.progress(100) if task end @@ -295,14 +296,16 @@ module ReachReports # end def self.get_report(type, id) - - case type - when /(?i)QMRF/ - report = ReachReports::QmrfReport.get(id) - when /(?i)QPRF/ - report = ReachReports::QprfReport.get(id) + report = nil + mysql_lite_retry(3) do + case type + when /(?i)QMRF/ + report = ReachReports::QmrfReport.get(id) + when /(?i)QPRF/ + report = ReachReports::QprfReport.get(id) + end + raise OpenTox::NotFoundError.new type+" report with id '#{id}' not found." unless report end - raise OpenTox::NotFoundError.new type+" report with id '#{id}' not found." unless report return report end -- cgit v1.2.3 From b542cfbd54901ad86d60fed03c8a05f9151f7616 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 31 May 2011 22:26:34 +0200 Subject: quit r instance after statistical test, will hopefully solve timeout problem --- report/environment.rb | 4 +--- report/report_content.rb | 1 + report/statistical_test.rb | 28 +++++++++++++++++++++++----- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/report/environment.rb b/report/environment.rb index a7d454f..72320a0 100755 --- a/report/environment.rb +++ b/report/environment.rb @@ -1,13 +1,11 @@ ['rubygems', 'logger', 'fileutils', 'sinatra', 'sinatra/url_for', 'rest_client', - 'yaml', 'fileutils', 'mime/types', 'abbrev', 'rinruby', + 'yaml', 'fileutils', 'mime/types', 'abbrev', 'rexml/document', 'ruby-plot', 'opentox-ruby' ].each do |g| require g end gem 'ruby-plot', "~>0.5.0" -#R.quit - module Reports end diff --git a/report/report_content.rb b/report/report_content.rb index d151a7d..30118cf 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -48,6 +48,7 @@ class Reports::ReportContent @xml_report.add_table(section_test, test_attribute.to_s+", significance-level: "+level.to_s+", num results: "+ test_matrix[:num_results].to_s, table, true, true) end + Reports::ReportStatisticalTest.quit_r end def add_predictions( validation_set, diff --git a/report/statistical_test.rb b/report/statistical_test.rb index 1e586e2..bee6241 100644 --- a/report/statistical_test.rb +++ b/report/statistical_test.rb @@ -1,29 +1,35 @@ #require "rubygems" #require "rinruby" -#R.quit module LIB class StatisticalTest - @@r = RinRuby.new(true,false) - # -1 -> array1 < array2 # 0 -> not difference # 1 -> array2 > array1 # def self.pairedTTest(array1, array2, significance_level=0.95) + + @@r = RinRuby.new(true,false) unless defined?(@@r) and @@r @@r.assign "v1",array1 @@r.assign "v2",array2 @@r.eval "ttest = t.test(v1,v2,paired=T)" t = @@r.pull "ttest$statistic" p = @@r.pull "ttest$p.value" - #@@r.quit if (1-significance_level > p) t else 0 end end + + def self.quit_r + begin + @@r.quit + @@r = nil + rescue + end + end end end @@ -68,9 +74,21 @@ module Reports LOGGER.debug "paired-t-testing "+attribute.to_s+" "+array1.inspect+" vs "+array2.inspect LIB::StatisticalTest.pairedTTest(array1, array2, significance_level) end + + def self.quit_r + LIB::StatisticalTest.quit_r + end + end end -#puts LIB::StatisticalTest.pairedTTest([1,2,3],[2,3,3]) +#t1 = Time.new +#10.times do +# puts LIB::StatisticalTest.pairedTTest([1,2,3,4,5,12,4,2],[2,3,3,3,56,3,4,5]) +#end +#LIB::StatisticalTest.quitR +#t2 = Time.new +#puts t2-t1 + -- cgit v1.2.3 From 7bbc70d38d81a5a57d90c525d90ccf7bf8f59426 Mon Sep 17 00:00:00 2001 From: Christoph Helma Date: Mon, 6 Jun 2011 16:54:56 +0000 Subject: halts (partially) substituted by OpenTox errors --- example.rb | 2 +- lib/active_record_setup.rb | 4 ++-- reach_reports/reach_application.rb | 8 ++++---- reach_reports/reach_service.rb | 2 +- report/report_application.rb | 2 +- report/report_persistance.rb | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/example.rb b/example.rb index b0b591e..636579e 100755 --- a/example.rb +++ b/example.rb @@ -80,7 +80,7 @@ class Example task.progress(10) log "upload dataset" - halt 400,"File not found: "+@@file.path.to_s unless File.exist?(@@file.path) + raise OpenTox::BadRequestError.new"File not found: "+@@file.path.to_s unless File.exist?(@@file.path) #data = File.read(@@file.path) #data_uri = OpenTox::RestClientWrapper.post(CONFIG[:services]["opentox-dataset"],{:content_type => @@file_type},data).chomp("\n") dataset = OpenTox::Dataset.create_from_csv_file(@@file.path,nil) diff --git a/lib/active_record_setup.rb b/lib/active_record_setup.rb index b43e692..5f081af 100755 --- a/lib/active_record_setup.rb +++ b/lib/active_record_setup.rb @@ -34,7 +34,7 @@ class ActiveRecord::Base unless self.column_names.include?(key) err = "no attribute found: '"+k.to_s+"'" # if $sinatra -# $sinatra.halt 400,err +# $sinatra.raise OpenTox::BadRequestError.newerr # else raise err # end @@ -47,4 +47,4 @@ class ActiveRecord::Base #puts "params after "+filter_params.inspect self.find(:all, :conditions => filter_params) end -end \ No newline at end of file +end diff --git a/reach_reports/reach_application.rb b/reach_reports/reach_application.rb index d914452..477adfb 100755 --- a/reach_reports/reach_application.rb +++ b/reach_reports/reach_application.rb @@ -25,7 +25,7 @@ require 'reach_reports/reach_service.rb' require "lib/format_util.rb" def extract_type(params) - halt 400, "illegal type, neither QMRF nor QPRF: "+params[:type] unless params[:type] && params[:type] =~ /(?i)Q(M|P)RF/ + raise OpenTox::BadRequestError.new "illegal type, neither QMRF nor QPRF: "+params[:type] unless params[:type] && params[:type] =~ /(?i)Q(M|P)RF/ params.delete("type") end @@ -96,7 +96,7 @@ get '/reach_report/:type/:id' do case request.env['HTTP_ACCEPT'].to_s when "application/rdf+xml" - halt 400, "application/rdf+xml not yet supported" + raise OpenTox::BadRequestError.new "application/rdf+xml not yet supported" owl = OpenTox::Owl.create(type+"Report",rep.report_uri) owl.set_data( rep.get_content.keys_to_rdf_format ) owl.rdf @@ -117,7 +117,7 @@ get '/reach_report/:type/:id' do content_type "application/x-yaml" rep.to_yaml else - halt 400, "MIME type '"+request.env['HTTP_ACCEPT'].to_s+"' not supported, valid Accept-Headers are \"application/rdf+xml\", \"application/x-yaml\", \"application/qmrf-xml\"." + raise OpenTox::BadRequestError.new "MIME type '"+request.env['HTTP_ACCEPT'].to_s+"' not supported, valid Accept-Headers are \"application/rdf+xml\", \"application/x-yaml\", \"application/qmrf-xml\"." end end @@ -128,7 +128,7 @@ post '/reach_report/:type/:id' do rep = ReachReports.get_report(type, params[:id]) input = request.env["rack.input"].read - halt 400, "no xml data specified" unless input && input.to_s.size>0 + raise OpenTox::BadRequestError.new "no xml data specified" unless input && input.to_s.size>0 LOGGER.debug "size of posted data: "+input.to_s.size.to_s ReachReports::QmrfReport.from_xml(rep,input) diff --git a/reach_reports/reach_service.rb b/reach_reports/reach_service.rb index 916a7a4..c93385c 100755 --- a/reach_reports/reach_service.rb +++ b/reach_reports/reach_service.rb @@ -289,7 +289,7 @@ module ReachReports # # report_content = get_report(type, id).get_content # keys.each do |k| -# $sinatra.halt 400, type+" unknown report property '#{key}'" unless report_content.is_a?(Hash) and report_content.has_key?(k) +# $sinatra.raise OpenTox::BadRequestError.new type+" unknown report property '#{key}'" unless report_content.is_a?(Hash) and report_content.has_key?(k) # report_content = report_content[k] # end # report_content diff --git a/report/report_application.rb b/report/report_application.rb index 3c8670a..debfe07 100755 --- a/report/report_application.rb +++ b/report/report_application.rb @@ -7,7 +7,7 @@ end def get_docbook_resource(filepath) perform do |rs| - halt 404,"not found: "+filepath unless File.exist?(filepath) + raise OpenTox::NotFoundError.new"not found: "+filepath unless File.exist?(filepath) types = MIME::Types.type_for(filepath) content_type(types[0].content_type) if types and types.size>0 and types[0] result = body(File.new(filepath)) diff --git a/report/report_persistance.rb b/report/report_persistance.rb index 78ae47b..e02387f 100755 --- a/report/report_persistance.rb +++ b/report/report_persistance.rb @@ -315,7 +315,7 @@ end # unless prop_names.include?(key) # err = "no attribute found: '"+k.to_s+"'" # if $sinatra -# $sinatra.halt 400,err +# $sinatra.raise OpenTox::BadRequestError.newerr # else # raise err # end -- cgit v1.2.3 From 654ff16ab0c63c9125785e9fe6546973cfe462a7 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 16 Jun 2011 23:52:31 +0200 Subject: add post forms --- reach_reports/reach_application.rb | 10 ++- report/report_application.rb | 12 +++- validation/validation_application.rb | 127 ++++++++++++++++++++--------------- 3 files changed, 91 insertions(+), 58 deletions(-) diff --git a/reach_reports/reach_application.rb b/reach_reports/reach_application.rb index d914452..aa64ea2 100755 --- a/reach_reports/reach_application.rb +++ b/reach_reports/reach_application.rb @@ -54,18 +54,19 @@ get '/reach_report/:type' do "All REACH reporting types: "+url_for("/reach_report",:full) description = "A list of "+type+" reports." - post_params = "" + post_command = nil case type when /(?i)QMRF/ related_links += "\n"+ "OpenTox version of QMRF editor: "+QMRF_EDITOR_URI description += "\n"+ "To create a QMRF report use the POST method." - post_params = [[[:model_uri]],[["Existing QMRF report, content-type application/qmrf-xml"]]] + post_command = OpenTox::PostCommand.new request.url,"Create QMRF report" + post_command.attributes << OpenTox::PostAttribute.new("model_uri") when /(?i)QPRF/ #TODO end - OpenTox.text_to_html ReachReports.list_reports(type),@subjectid,related_links,description,post_params + OpenTox.text_to_html ReachReports.list_reports(type),@subjectid,related_links,description,post_command else content_type "text/uri-list" ReachReports.list_reports(type) @@ -78,6 +79,9 @@ post '/reach_report/:type' do content_type "text/uri-list" LOGGER.info "creating "+type+" report "+params.inspect + raise OpenTox::BadRequestError.new "model_uri missing" if type=~/(?i)QMRF/ and + params[:model_uri]!=nil and params[:model_uri].to_s.size==0 + #puts "creating "+type+" report "+params.inspect result_uri = ReachReports.create_report(type,params,@subjectid,request.env["rack.input"]) diff --git a/report/report_application.rb b/report/report_application.rb index 3c8670a..f7780c3 100755 --- a/report/report_application.rb +++ b/report/report_application.rb @@ -61,8 +61,17 @@ get '/report/:report_type' do description = "A list of all "+params[:report_type]+" reports. To create a report, use the POST method." post_params = [[:validation_uris]] + + post_command = OpenTox::PostCommand.new request.url,"Create validation report" + val_uri_description = params[:report_type]=="algorithm_comparison" ? "Separate multiple uris with ','" : nil + # trick for easy report creation + # if searching for a report, ?validation="uri" or ?crossvalidaiton="uri" is given as search param + # use this (search param has equal name as report type) as default value for validation_uri + post_command.attributes << OpenTox::PostAttribute.new("validation_uris",true,params[params[:report_type]],val_uri_description) + post_command.attributes << OpenTox::PostAttribute.new("identifier",true,nil,"Specifiy one identifier for each uri, separated with ','") if + params[:report_type]=="algorithm_comparison" content_type "text/html" - OpenTox.text_to_html rs.get_all_reports(params[:report_type], params),@subjectid,related_links,description,post_params + OpenTox.text_to_html rs.get_all_reports(params[:report_type], params),@subjectid,related_links,description,post_command else content_type "text/uri-list" rs.get_all_reports(params[:report_type], params) @@ -116,6 +125,7 @@ delete '/report/:type/:id' do end post '/report/:type' do + raise OpenTox::BadRequestError.new "validation_uris missing" unless params[:validation_uris].to_s.size>0 task = OpenTox::Task.create("Create report",url_for("/report/"+params[:type], :full)) do |task| #,params perform do |rs| rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil, diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 5ae6bda..bd55d4c 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -17,9 +17,16 @@ get '/crossvalidation/?' do description = "A list of all crossvalidations.\n"+ "Use the POST method to perform a crossvalidation." - post_params = [[:dataset_uri,:algorithm_uri,:prediction_feature,[:num_folds,10],[:random_seed,1],[:stratified,false],[:algorithm_params,""]]] + post_command = OpenTox::PostCommand.new request.url,"Perform crossvalidation" + post_command.attributes << OpenTox::PostAttribute.new("algorithm_uri") + post_command.attributes << OpenTox::PostAttribute.new("dataset_uri") + post_command.attributes << OpenTox::PostAttribute.new("prediction_feature") + post_command.attributes << OpenTox::PostAttribute.new("algorithm_params",false,nil,"Params used for model building, separate with ';', example: param1=v1;param2=v2") + post_command.attributes << OpenTox::PostAttribute.new("num_folds",false,"10") + post_command.attributes << OpenTox::PostAttribute.new("random_seed",false,"1","An equal random seed value ensures the excact same random dataset split.") + post_command.attributes << OpenTox::PostAttribute.new("stratified",false,"false","Stratification ensures an equal class-value spread in folds.") content_type "text/html" - OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_params + OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_command else content_type "text/uri-list" uri_list @@ -27,14 +34,14 @@ get '/crossvalidation/?' do end post '/crossvalidation/?' do - task = OpenTox::Task.create( "Perform crossvalidation", url_for("/crossvalidation", :full) ) do |task| #, params - LOGGER.info "creating crossvalidation "+params.inspect - raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] - raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri] - raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature] - raise OpenTox::BadRequestError.new "illegal param-value num_folds: '"+params[:num_folds].to_s+"', must be integer >1" unless params[:num_folds]==nil or - params[:num_folds].to_i>1 + LOGGER.info "creating crossvalidation "+params.inspect + raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri].to_s.size>0 + raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri].to_s.size>0 + raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0 + raise OpenTox::BadRequestError.new "illegal param-value num_folds: '"+params[:num_folds].to_s+"', must be integer >1" unless params[:num_folds]==nil or + params[:num_folds].to_i>1 + task = OpenTox::Task.create( "Perform crossvalidation", url_for("/crossvalidation", :full) ) do |task| #, params cv_params = { :dataset_uri => params[:dataset_uri], :algorithm_uri => params[:algorithm_uri], :loo => "false" } @@ -71,14 +78,13 @@ post '/crossvalidation/cleanup/?' do end post '/crossvalidation/loo/?' do + LOGGER.info "creating loo-crossvalidation "+params.inspect + raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri].to_s.size>0 + raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri].to_s.size>0 + raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0 + raise OpenTox::BadRequestError.new "illegal param: num_folds, stratified, random_seed not allowed for loo-crossvalidation" if params[:num_folds] or + params[:stratifed] or params[:random_seed] task = OpenTox::Task.create( "Perform loo-crossvalidation", url_for("/crossvalidation/loo", :full) ) do |task| #, params - LOGGER.info "creating loo-crossvalidation "+params.inspect - raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] - raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri] - raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature] - raise OpenTox::BadRequestError.new "illegal param: num_folds, stratified, random_seed not allowed for loo-crossvalidation" if params[:num_folds] or - params[:stratifed] or params[:random_seed] - cv_params = { :dataset_uri => params[:dataset_uri], :algorithm_uri => params[:algorithm_uri], :loo => "true" } @@ -104,9 +110,13 @@ get '/crossvalidation/loo/?' do description = "A list of all leave one out crossvalidations.\n"+ "Use the POST method to perform a crossvalidation." - post_params = [[:dataset_uri,:algorithm_uri,:prediction_feature,[:algorithm_params,""]]] + post_command = OpenTox::PostCommand.new request.url,"Perform leave-one-out-crossvalidation" + post_command.attributes << OpenTox::PostAttribute.new("algorithm_uri") + post_command.attributes << OpenTox::PostAttribute.new("dataset_uri") + post_command.attributes << OpenTox::PostAttribute.new("prediction_feature") + post_command.attributes << OpenTox::PostAttribute.new("algorithm_params",false,nil,"Params used for model building, separate with ';', example: param1=v1;param2=v2") content_type "text/html" - OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_params + OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_command else content_type "text/uri-list" uri_list @@ -256,7 +266,8 @@ end post '/test_set_validation' do LOGGER.info "creating test-set-validation "+params.inspect - if params[:model_uri] and params[:test_dataset_uri] and !params[:training_dataset_uri] and !params[:algorithm_uri] + if params[:model_uri].to_s.size>0 and params[:test_dataset_uri].to_s.size>0 and + params[:training_dataset_uri].to_s.size==0 and params[:algorithm_uri].to_s.size==0 task = OpenTox::Task.create( "Perform test-set-validation", url_for("/", :full) ) do |task| #, params v = Validation::Validation.create :validation_type => "test_set_validation", :model_uri => params[:model_uri], @@ -290,9 +301,13 @@ get '/test_set_validation' do description = "A list of all test-set-validations.\n"+ "To perform a test-set-validation use the POST method." - post_params = [[:model_uri, :test_dataset_uri, [:test_target_dataset_uri,"same-as-test_dataset_uri"], [:prediction_feature, "dependent-variable-of-model"]]] + post_command = OpenTox::PostCommand.new request.url,"Perform test-set-validation" + post_command.attributes << OpenTox::PostAttribute.new("model_uri") + post_command.attributes << OpenTox::PostAttribute.new("test_dataset_uri") + post_command.attributes << OpenTox::PostAttribute.new("test_target_dataset_uri",false,nil,"Specify if target endpoint values are not available in test dataset.") + post_command.attributes << OpenTox::PostAttribute.new("prediction_feature",false,nil,"Default is 'dependentVariables' of the model.") content_type "text/html" - OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_params + OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_command else content_type "text/uri-list" uri_list @@ -301,7 +316,8 @@ end post '/training_test_validation/?' do LOGGER.info "creating training-test-validation "+params.inspect - if params[:algorithm_uri] and params[:training_dataset_uri] and params[:test_dataset_uri] and params[:prediction_feature] and !params[:model_uri] + if params[:algorithm_uri].to_s.size>0 and params[:training_dataset_uri].to_s.size>0 and + params[:test_dataset_uri].to_s.size>0 and params[:prediction_feature].to_s.size>0 and params[:model_uri].to_s.size==0 task = OpenTox::Task.create( "Perform training-test-validation", url_for("/", :full) ) do |task| #, params v = Validation::Validation.create :validation_type => "training_test_validation", :algorithm_uri => params[:algorithm_uri], @@ -335,14 +351,15 @@ get '/training_test_validation' do description = "A list of all training-test-validations.\n"+ "To perform a training-test-validation use the POST method." - post_params = [[:algorithm_uri, - :training_dataset_uri, - :test_dataset_uri, - [:test_target_dataset_uri,"same-as-test_dataset_uri"], - :prediction_feature, - [:algorithm_params, ""]]] + post_command = OpenTox::PostCommand.new request.url,"Perform training-test-validation" + post_command.attributes << OpenTox::PostAttribute.new("algorithm_uri") + post_command.attributes << OpenTox::PostAttribute.new("training_dataset_uri") + post_command.attributes << OpenTox::PostAttribute.new("test_dataset_uri") + post_command.attributes << OpenTox::PostAttribute.new("test_target_dataset_uri",false,nil,"Specify if target endpoint values are not available in test dataset.") + post_command.attributes << OpenTox::PostAttribute.new("prediction_feature") + post_command.attributes << OpenTox::PostAttribute.new("algorithm_params",false,nil,"Params used for model building, separate with ';', example: param1=v1;param2=v2") content_type "text/html" - OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_params + OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_command else content_type "text/uri-list" uri_list @@ -350,19 +367,21 @@ get '/training_test_validation' do end post '/bootstrapping' do + LOGGER.info "performing bootstrapping validation "+params.inspect + raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri].to_s.size>0 + raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri].to_s.size>0 + raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0 task = OpenTox::Task.create( "Perform bootstrapping validation", url_for("/bootstrapping", :full) ) do |task| #, params - LOGGER.info "performing bootstrapping validation "+params.inspect - raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] - raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri] - raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature] - params.merge!( Validation::Util.bootstrapping( params[:dataset_uri], params[:prediction_feature], @subjectid, params[:random_seed], OpenTox::SubTask.create(task,0,33)) ) + LOGGER.info "params after bootstrapping: "+params.inspect v = Validation::Validation.create :validation_type => "bootstrapping", :test_target_dataset_uri => params[:dataset_uri], :prediction_feature => params[:prediction_feature], - :algorithm_uri => params[:algorithm_uri] + :algorithm_uri => params[:algorithm_uri], + :training_dataset_uri => params[:training_dataset_uri], + :test_dataset_uri => params[:test_dataset_uri] v.subjectid = @subjectid v.validate_algorithm( params[:algorithm_params], OpenTox::SubTask.create(task,33,100)) v.validation_uri @@ -385,13 +404,14 @@ get '/bootstrapping' do description = "A list of all bootstrapping-validations.\n"+ "To perform a bootstrapping-validation use the POST method." - post_params = [[:algorithm_uri, - :dataset_uri, - :prediction_feature, - [:algorithm_params, ""], - [:random_seed, 1]]] + post_command = OpenTox::PostCommand.new request.url,"Perform bootstrapping-validation" + post_command.attributes << OpenTox::PostAttribute.new("algorithm_uri") + post_command.attributes << OpenTox::PostAttribute.new("dataset_uri") + post_command.attributes << OpenTox::PostAttribute.new("prediction_feature") + post_command.attributes << OpenTox::PostAttribute.new("algorithm_params",false,nil,"Params used for model building, separate with ';', example: param1=v1;param2=v2") + post_command.attributes << OpenTox::PostAttribute.new("random_seed",false,"1","An equal random seed value ensures the excact same random dataset split.") content_type "text/html" - OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_params + OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_command else content_type "text/uri-list" uri_list @@ -399,13 +419,11 @@ get '/bootstrapping' do end post '/training_test_split' do - + LOGGER.info "creating training test split "+params.inspect + raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri].to_s.size>0 + raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri].to_s.size>0 + raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0 task = OpenTox::Task.create( "Perform training test split validation", url_for("/training_test_split", :full) ) do |task| #, params - LOGGER.info "creating training test split "+params.inspect - raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] - raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri] - raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature] - params.merge!( Validation::Util.train_test_dataset_split(params[:dataset_uri], params[:prediction_feature], @subjectid, params[:split_ratio], params[:random_seed], OpenTox::SubTask.create(task,0,33))) v = Validation::Validation.create :validation_type => "training_test_split", @@ -437,14 +455,15 @@ get '/training_test_split' do description = "A list of all training-test-split-validations.\n"+ "To perform a training-test-split-validation use the POST method." - post_params = [[:algorithm_uri, - :dataset_uri, - :prediction_feature, - [:algorithm_params, ""], - [:random_seed, 1], - [:split_ratio, 0.66]]] + post_command = OpenTox::PostCommand.new request.url,"Perform training-test-split-validation" + post_command.attributes << OpenTox::PostAttribute.new("algorithm_uri") + post_command.attributes << OpenTox::PostAttribute.new("dataset_uri") + post_command.attributes << OpenTox::PostAttribute.new("prediction_feature") + post_command.attributes << OpenTox::PostAttribute.new("algorithm_params",false,nil,"Params used for model building, separate with ';', example: param1=v1;param2=v2") + post_command.attributes << OpenTox::PostAttribute.new("random_seed",false,"1","An equal random seed value ensures the excact same random dataset split.") + post_command.attributes << OpenTox::PostAttribute.new("split_ratio",false,"0.66","A split ratio of 0.66 implies that two thirds of the compounds are used for training.") content_type "text/html" - OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_params + OpenTox.text_to_html uri_list,@subjectid,related_links,description,post_command else content_type "text/uri-list" uri_list -- cgit v1.2.3 From 5d66676376bd491344f06294a0921575aa9c39f6 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 17 Jun 2011 00:27:44 +0200 Subject: report stylesheet moved from ortona to validation service --- report/report_format.rb | 4 +- resources/simple_ot_stylesheet.css | 2291 ++++++++++++++++++++++++++++++++++++ 2 files changed, 2294 insertions(+), 1 deletion(-) create mode 100644 resources/simple_ot_stylesheet.css diff --git a/report/report_format.rb b/report/report_format.rb index 67abc1e..d64bf57 100644 --- a/report/report_format.rb +++ b/report/report_format.rb @@ -4,6 +4,8 @@ ENV['JAVA_HOME'] = "/usr/bin" unless ENV['JAVA_HOME'] ENV['PATH'] = ENV['JAVA_HOME']+":"+ENV['PATH'] unless ENV['PATH'].split(":").index(ENV['JAVA_HOME']) ENV['SAXON_JAR'] = "saxonhe9-2-0-3j/saxon9he.jar" unless ENV['SAXON_JAR'] +OT_STYLESHEET = File.join(CONFIG[:services]["opentox-validation"],"resources/simple_ot_stylesheet.css") + # = Reports::ReportFormat # # provides functions for converting reports from xml to other formats @@ -60,7 +62,7 @@ module Reports::ReportFormat end def self.format_report_to_html(directory, xml_filename, html_filename, css_style_sheet) - css_style_sheet = "http://opentox.informatik.uni-freiburg.de/simple_ot_stylesheet.css" unless css_style_sheet + css_style_sheet = OT_STYLESHEET unless css_style_sheet css = css_style_sheet ? "--stringparam html.stylesheet "+URI.encode(css_style_sheet.to_s) : nil cmd = "xsltproc "+css.to_s+" "+ENV['REPORT_XSL']+" "+File.join(directory,xml_filename.to_s)+" > "+File.join(directory,html_filename.to_s) diff --git a/resources/simple_ot_stylesheet.css b/resources/simple_ot_stylesheet.css new file mode 100644 index 0000000..4c4c072 --- /dev/null +++ b/resources/simple_ot_stylesheet.css @@ -0,0 +1,2291 @@ +/* - base.css - */ +@media screen { +/* http://www.opentox.org/portal_css/base.css?original=1 */ +/* */ +/* */ +body { +font: 75% Verdana,Arial,Helvetica,sans-serif; +background-color: White; +color: #333; +margin: 1.0em 1.0em 1.0em 1.0em; +padding: 0; +} +table { +font-size: 100%; +} +table td{ +padding: 3px; +} +a { +color: #5D308A; +background-color: transparent; +} +img { +border: none; +vertical-align: middle; +} +p { +margin: 0.25em 0 0.25em 0; +line-height: 1.5em; +} +p img { +border: none; +margin: 0; +} +hr { +border: 0; +height: 1px; +color: #000; +background-color: #000; +margin: 0.5em 0 1em 0; +} +h1, h2, h3, h4, h5, h6 { +color: #333; +font-family: Arial,Helvetica,Verdana,Geneva,sans-serif; +margin: 0.75em 0 0.25em 0; +} +h1 a, +h2 a, +h3 a, +h4 a, +h5 a, +h6 a { +color: #333 ! important; +text-decoration: none; +} +h1 { +font-size: 160%; +margin: 2em 0 0.25em 0; +} +h2 { +font-size: 150%; +margin: 2em 0 0.25em 0; +} +h3 { +font-size: 125%; +border-bottom: none; +font-weight: bold; +} +h4 { +font-size: 110%; +border-bottom: none; +font-weight: bold; +} +h5 { +font-size: 100%; +border-bottom: none; +font-weight: bold; +} +h6 { +font-size: 0.9em; +border-bottom: none; +font-weight: bold; +} +ul { +line-height: 1.5em; +padding: 0; +} +ol { +line-height: 1.5em; +padding: 0; +} +li { +margin-bottom: 0.5em; +} +dt { +font-weight: bold; +} +dd { +line-height: 1.5em; +margin-bottom: 1em; +} +abbr, acronym, .explain { +border-bottom: 1px dotted #333; +color: #333; +background-color: transparent; +cursor: help; +} +abbr .explain { +border-bottom: none; +} +q { +font-family: Baskerville, Georgia, serif; +font-style: italic; +font-size: 120%; +} +blockquote { +padding-left: 0.5em; +margin-left: 0; +border-left: 4px solid #000; +color: #666666; +} +code, tt { +font-family: Monaco, "Courier New", Courier, monospace; +font-size: 120%; +color: #333; +background-color: #5D308A; +padding: 0 0.1em; +} +pre { +font-family: Monaco, "Courier New", Courier, monospace; +font-size: 100%; +padding: 1em; +border: 1px solid #000; +color: #333; +background-color: #5D308A; +overflow: auto; +} +ins { +color: green; +text-decoration: none; +} +del { +color: red; +text-decoration: line-through; +} +/* */ + +} + + +/* - public.css - */ +@media screen { +/* http://www.opentox.org/portal_css/public.css?original=1 */ +/* */ +body.largeText { font-size: 95%; } +body.smallText { font-size: 60%; } +/* */ +h2 { +font-weight: normal; +} +/* */ +body.kssActive h2.inlineEditable:hover, +body.kssActive h1.inlineEditable:hover { +padding-bottom: 1px; +} +h3, h4, h5, h6 { +font-weight: bold; +} +.documentFirstHeading { +margin-top: 0; +} +.documentContent { +background: White; +} +.documentContent ul { +list-style-image: url(http://www.opentox.org/bullet.gif); +list-style-type: square; +margin: 0.5em 0 0 1.5em; +} +.documentContent ol { +margin: 0.5em 0 0 2.5em; +} +#visual-portal-wrapper { +padding: 0; +} +/* */ +#portal-logo img { +border: 0; +padding: 0; +} +/* */ +#portal-skinswitcher { +} +#portal-skinswitcher a { +display: block; +float: left; +} +#portal-top { +/* */ +margin: 0; +padding: 0; +background-color: transparent; +} +/* */ +#portal-siteactions { +background-color: transparent; +list-style-image: none; +list-style-type: none; +height: auto; +line-height: normal; +} +#portal-siteactions li { +display: inline; +} +#portal-siteactions li a { +background-color: transparent; +height: auto; +text-decoration: none; +text-transform: none; +} +/* */ +#portal-searchbox { +float: right; +clear: right; +background-color: transparent; +text-align: right; +text-transform: none; +white-space: nowrap; +z-index: 2; +} +#portal-advanced-search { +margin-top: 0.2em; +clear: both; +} +#portal-advanced-search a { +color: #666666; +text-decoration: none; +text-transform: none; +} +/* */ +dl.searchResults dt { +font-size: 140%; +font-weight: normal; +} +form.searchPage { +text-align: center; +} +input.searchPage { +font-size: 200% !important; +} +form.searchPage input.searchButton { +background-position:5px 7px; +padding:1px 10px 1px 25px; +} +/* */ +.LSRes { +font-family: Verdana,Arial,Helvetica,sans-serif; +visibility: visible; +color: #fff; +background-color: White; +vertical-align: middle; +display:block; +list-style-image: none; +list-style-type: none; +text-align: left; +min-width: 16.5em; +text-transform: none; +margin-left: 0; +line-height: 1.1em; +} +#LSHighlight, +.LSHighlight { +background-color: #5D308A; +border: 1px solid #000; +} +.LSRow { +border: 1px solid White; +white-space: normal; +padding:0; +margin: 0; +list-style-image: none; +list-style-type: none; +} +.LSRow a { +text-decoration: none; +font-weight:bold; +white-space:nowrap +} +.LSDescr { +color: #666666; +text-transform: none; +padding-left:2.1em; +margin-top:-0.1em; +} +.LSResult { +position: relative; +display: block; +text-align: right; +padding-top: 5px; +margin: 0; +left: 3px; +z-index: 3; +} +.LSShadow { +position: relative; +text-align: right; +} +.livesearchContainer { +background-color: White; +margin-top: 0; +padding: 0 !important; +position: absolute; +right: 0px; +/* */ +top: 0; +white-space: normal; +font-family: Verdana,Arial,Helvetica,sans-serif; +visibility: visible; +text-align: left; +border: 1px solid #000; +width: 30em; +text-transform: none; +} +* html .livesearchContainer { +padding: 1px !important; +padding-top: 0 !important; +background-color: #000; +border: 0; +} +#livesearchLegend { +line-height: 1em; +margin-top: -2em; +margin-left: -0.1em; +border: 1px solid #000; +border-bottom: 0; +} +* html #livesearchLegend { +margin-top: -1.9em; +margin-left: -8px; +position: relative; +} +/* */ +.LSIEFix { +background-color: White; +padding: 0.5em !important; +z-index: 20; +} +.LSBox { +clear: left; +float: left; +text-align: right; +padding-right: 1px; +display:block; +} +#LSNothingFound { +text-align: center; +padding: 2px; +} +.LSBox label { +font-weight: normal; +} +/* */ +#portal-globalnav { +white-space: nowrap; +list-style: none; +height: auto; +line-height: normal; +} +#portal-globalnav li { +display: inline; +} +#portal-globalnav li a { +/* */ +background-color: transparent; +color: #fff; +height: auto; +text-decoration: none; +} +#portal-globalnav li.selected a { +/* */ +color: #fff; +} +#portal-globalnav li a:hover { +color: #fff; +} +#portal-languageselector { +float:right; +} +#portal-languageselector li { +display: inline; +} +#portal-personaltools { +/* */ +line-height: 1.6em; +color: #333; +margin: 0; +text-align: right; +text-transform: none; +list-style: none; +} +#portal-personaltools .portalUser { +background: transparent url(http://www.opentox.org/user.gif) center left no-repeat; +padding-left: 18px; +} +#portal-personaltools .portalNotLoggedIn { +/* */ +color: #333; +padding: 0; +background: transparent; +background-image: none; +} +#portal-personaltools li { +color: #fff; +margin-left: 1em; +display: inline; +} +#portal-personaltools li a { +text-decoration: none; +color: #fff; +} +#portal-personaltools .visualIconPadding { +padding-left: 10px; +} +.visualCaseSensitive { +text-transform: none; +} +#portal-breadcrumbs a { +text-decoration: none; +} +.breadcrumbSeparator { +font-size: 120%; +} +.addFavorite { +vertical-align: bottom; +} +#content-news h1 { +margin-bottom: 1em; +} +.newsItem { +margin-bottom: 1em; +border-bottom: 1px solid #000; +} +.newsImage { +border: 1px solid #ccc; +} +.newsImageContainer { +float:right; +margin: 0 0 0.5em 1em; +width: 202px; +} +.newsContent { +padding: 0 1em 1em 1em; +} +.newsContent ul, +.newsContent li { +display: block; +list-style: none; +list-style-image: none; +margin: 0; +padding: 0; +} +.newsAbout { +display: block; +color: #666666; +font-size: 0.9em; +padding: 0; +margin-top: 0; +list-style: none; +list-style-image: none; +float: right; +text-align: right; +} +.newsAbout li { +display: inline; +} +.newsFooter { +} +.newsFooter li { +display: inline; +margin: 0 1em 0 0; +} +.documentActions { +margin: 1em 0; +padding: 0; +text-align: right; +} +.documentActions ul { +margin: 0; +padding: 0 0.5em; +display: block; +list-style-type: none; +list-style-image: none; +} +.documentActions li { +display: inline; +margin: 0 0.5em; +padding: 0 0.25em; +background-color: White; +} +.documentActions a { +text-decoration: none; +} +/* */ +dl.portalMessage { +font-size: 0.9em; +} +dl.portalMessage a { +color: black; +border: none; +text-decoration: underline; +} +dl.portalMessage dt { +background-color: #996; +border: 1px solid #996; +font-weight: bold; +float: left; +margin: 0 0.5em 0 0; +padding: 0.5em 0.75em; +color: White; +line-height: 1.25em; +} +dl.portalMessage dd { +background-color: #ffffe3; +border: 1px solid #996; +padding: 0.5em 0.5em; +margin: 0; +line-height: 1.25em; +} +dl.warning dt { +background-color: #d80; +border: 1px solid #d80; +} +dl.error dt { +background-color: #d00; +border-color: #d00; +} +dl.warning dd { +background-color: #fd7; +border-color: #d80; +} +dl.error dd { +background-color: #fd7; +border-color: #d80; +} +.documentDescription { +/* */ +font-weight: bold; +display: block; +margin: 0em 0em 0.5em 0em; +line-height: 1.5em; +} +.documentByLine { +font-size: 0.9em; +font-weight: normal; +color: #666666; +margin-bottom: 0.5em; +} +dl.searchResults span.documentByLine { +display: block; +} +#category ul { +list-style-image: none; +list-style-type: none; +display: inline; +margin: 0; +} +#category ul li { +display: inline; +} +.even { +background-color: #DDDDDD; +} +.odd { +background-color: transparent; +} +.discussion { +margin-top: 1em; +} +.visualHighlight { +background-color: #ffc; +} +.discreet { +color: #666666; +font-size: 0.9em; +font-weight: normal; +} +.pullquote { +padding: 0 1em 0 1em; +margin: 0 0 1em 1em; +font-weight: bold; +float: right; +width: 35%; +clear: right; +background-color: White; +border-left: 4px solid #000; +} +.callout { +font-weight: bold; +padding: 0px 1em; +} +.notify, +.documentEditable * .notify { +border: 1px solid #ffa500; +} +.card { +background-color: #5D308A; +border-color: #000; +border-width: 1px; +border-style: solid; +float: left; +margin: 1em; +text-align: center; +width: 110px; +padding: 1em 0; +} +.card a { +text-decoration: none; +} +.portrait { +background-color: #5D308A; +border-color: #000; +border-width: 1px; +border-style: solid; +font-size: 0.9em; +margin: 0.5em; +padding: 1em 0 0.5em 0; +text-align: center; +width: 100px; +} +.portraitPhoto { +border: 1px solid black; +} +/* */ +table.listing, +.stx table { +/* */ +border-collapse: collapse; +border-left: 1px solid #000; +border-bottom: 1px solid #000; +font-size: 0.9em; +margin: 1em 0em 1em 0em; +} +table.listing th, +.stx table th { +border-top: 1px solid #000; +border-bottom: 1px solid #000; +border-right: 1px solid #000; +font-weight: normal; +padding: 0.25em 0.5em; +text-transform: none; +} +table.listing .top { +border-left: 1px solid White; +border-top: 1px solid White ! important; +border-right: 1px solid White ! important; +text-align: right ! important; +padding: 0em 0em 1em 0em; +} +table.listing .listingCheckbox { +text-align: center; +} +table.listing td, +.stx table td { +border-right: 1px solid #000; +padding: 0.25em 0.5em; +} +table.listing a { +text-decoration: none; +} +table.listing a:hover { +text-decoration: underline; +} +table.listing img { +vertical-align: middle; +} +table.listing td a label, +.stx table td a label { +cursor: pointer; +} +/* */ +table.vertical th { +padding: 0.5em; +} +table.vertical td { +border-top: 1px solid #000; +padding: 0.5em; +} +/* */ +table.grid td { +border: 1px solid #000; +padding: 0.5em; +} +/* */ +table.plain, +table.plain td, +table.plain th { +border: 1px solid #ccc; +padding: 0.5em; +border-collapse: collapse; +} +/* */ +table.plainnoboder, +table.plainnoboder td, +table.plainnoboder th { +border: none; +padding: 0.5em; +border-collapse: collapse; +} +/* */ +.listingBar { +border-style: solid; +border-width: 1px; +padding: 0em 1em; +text-align: center; +text-transform: none; +vertical-align: top; +margin: 1em 0em; +font-size: 94%; +clear: both; +} +.listingBar span.previous, +.listingPrevious { +text-align: left; +float: left; +margin-right: 1em; +} +.listingBar span.next, +.listingNext { +text-align: right; +float: right; +margin-left: 1em; +} +.listingBar img { +vertical-align: middle; +} +.listingBar a { +text-decoration: none; +} +.tileItem { +padding-top: 0.5em; +margin-top: 0.5em; +} +.tileHeadline { +border: none; +font-size: 110%; +font-weight: bold; +} +.tileHeadline a { +text-decoration: none; +} +.tileBody { +margin-bottom: 0.5em; +} +.eventDetails { +float: right; +width: 20em; +clear: right; +} +/* */ +/* */ +ul.visualNoMarker, +ol.visualNoMarker { +list-style-type: none; +list-style-image: none; +margin: 0.5em 0 0 0; +line-height: 1em; +} +ul.discreet { +list-style-image: none; +list-style-type: disc; +} +textarea.proportional { +font: 100% Verdana,Arial,Helvetica,sans-serif; +} +.productCredits { +text-align: right; +font-size: 0.9em; +clear: both; +font-weight: normal; +color: #666666; +} +#portal-footer { +float: none; +line-height: 1.2em; +text-align: center; +} +#portal-footer p { +margin: 0.25em 0; +} +#portal-footer a { +text-decoration: none; +color: #5D308A; +border: none; +} +#portal-footer a:visited { +color: #5D308A; +} +#portal-footer a:hover { +text-decoration: underline; +} +#portal-colophon { +float: none; +margin: 0 0 1em 0; +padding: 0 0 1em 0; +text-align: center; +color: #666; +} +#portal-colophon ul { +list-style-image: none; +list-style-type: none; +} +#portal-colophon ul li { +display: inline !important; +font-size: 0.9em; +padding: 0 0.75em; +} +#portal-colophon ul li a { +text-decoration: none; +border-bottom: 1px #ccc solid; +color: #666; +} +.feedButton { +display: block; +float: right; +margin-top: 1px; +} +.poweredBy { +display: block; +clear: both; +font-size: 0.9em; +font-weight: normal; +color: #666666; +text-align: right; +} +/* */ +#portal-sitemap { +list-style: none; +list-style-image: none; +margin: 0; +font-size: 90%; +border: none; +} +#portal-sitemap .navTreeLevel1 { +padding-left: 1em; +border-left: 0.5em solid #000; +margin: 0 0 0 0.5em; +} +#portal-sitemap .navTreeLevel2 { +padding-left: 1em; +border-left: 0.5em solid #5D308A; +} +/* */ +.photoAlbumEntry { +float: left; +height: 185px; +width: 143px; +margin: 0em; +padding: 0px 6px 0px 9px; +text-align: center; +background-image: url('http://www.opentox.org/polaroid-single.png'); +background-repeat: no-repeat; +} +.photoAlbumEntry img { +border: 1px solid #ccc; +display: block; +margin: 0 auto; +} +.photoAlbumEntryWrapper { +height: 130px; +width: 128px; +margin-bottom: 7px; +} +.photoAlbumEntry a { +display: block; +text-decoration: none; +font-size: 0.9em; +height: 169px; +width: 130px; +margin: 16px auto 0px; +} +.photoAlbumFolder { +background-image: url('http://www.opentox.org/polaroid-multi.png'); +background-repeat: no-repeat; +} +.photoAlbumEntryTitle { +color: #666666; +display: block; +overflow: hidden; +width: 128px; +height: 3.6em; +} +/* */ +a.link-parent { +display: block; +background: transparent url(http://www.opentox.org/arrowUp.gif) 4px 5px no-repeat; +padding: 1px 0px 10px 16px; +font-size: 0.9em; +text-decoration: none; +} +#content .link-category { +color: #74ae0b !important; +} +#content .link-user { +background: transparent url(http://www.opentox.org/user.gif) 0 1px no-repeat; +padding: 1px 0px 1px 16px; +} +#content .link-comment { +background: transparent url(http://www.opentox.org/discussionitem_icon.gif) center left no-repeat; +padding: 1px 0px 1px 16px !important; /* */ +} +#content .link-anchor { +color: #666666; +text-decoration: none; +font-weight: normal; +} +#content .link-presentation { +font-size: 90%; +text-align: center; +} +#content .link-wiki-add { +color: red; +} +/* */ +.visualGhosted { +opacity: 0.2; +} +/* */ +body.fullscreen #portal-logo, +body.fullscreen #portal-siteactions { +display: none; +} +body.fullscreen #portal-globalnav { +margin-top: 4em; +} +body.fullscreen #portal-searchbox { +margin: 0.5em 2em 0 0.5em; +padding: 0; +position: relative; +z-index: 3; +} +/* */ +.image-left { +float: left; +clear: both; +margin: 0.5em 1em 0.5em 0; +} +.image-inline { +float: none; +} +.image-right { +float: right; +clear: both; +margin: 0.5em; +} +dd.image-caption { +text-align:left; +padding: 0; margin:0; +} +dl.captioned { +padding: 10px; +} +/* */ +#dashboard-info-message { +padding-top: 0.5em; +} +#dashboard { +width: 68em; +} +#dashboard-portlets1, +#dashboard-portlets2, +#dashboard-portlets3 +{ +float:left; +width:16em; +padding:0.7em 1.3em 0 0; +} +#dashboard-portlets4 { +float:left; +width:16em; +padding-top:0.7em; +} +#dashboard-portlets1 a, +#dashboard-portlets2 a, +#dashboard-portlets3 a, +#dashboard-portlets4 a { +border-bottom:medium none; +} +#dashboard-portlets1 dl.portlet, +#dashboard-portlets2 dl.portlet, +#dashboard-portlets3 dl.portlet, +#dashboard-portlets4 dl.portlet { +margin-bottom:1.5em; +} +div.managedPortlet.portlet { +border-bottom:none; +} +#dashboard select { +width:100%; +} +.portletAssignments { +margin-top:1.5em; +} +#dashboard-portlets1 div.managedPortlet a, +#dashboard-portlets2 div.managedPortlet a, +#dashboard-portlets3 div.managedPortlet a, +#dashboard-portlets4 div.managedPortlet a { +text-decoration: none; +color: #fff; +border-bottom:1px solid #fff; +} +#dashboard-portlets1 div.managedPortlet span a, +#dashboard-portlets2 div.managedPortlet span a, +#dashboard-portlets3 div.managedPortlet span a, +#dashboard-portlets4 div.managedPortlet span a{ +border-bottom:none; +} +#dashboard-actions { +float:right; +} +#dashboard-actions ul { +list-style-image:none; +list-style-position:outside; +list-style-type:none; +margin-top:0; +} +#dashboard-actions ul li { +display:inline; +padding-left:0.7em; +} +#dashboard-actions ul li.portalUser { +background:transparent url(http://www.opentox.org/user.gif) no-repeat scroll left center; +padding-left:18px; +} +/* */ +.section div { +padding-top:0; +padding-bottom:0; +} +/* */ +/* */ + +} + + +/* - columns.css - */ +@media screen { +/* http://www.opentox.org/portal_css/columns.css?original=1 */ +/* */ +#portal-columns { +width: 100% !important; +border-collapse: collapse; +border-spacing: 0; +} +#portal-column-one { +vertical-align: top; +width: 16em; +border-collapse: collapse; +padding: 0; +} +#portal-column-content { +vertical-align: top; +border-collapse: collapse; +padding: 1em 1em 0 1em; +margin: 0em 0em 2em 0em; +} +#portal-column-two { +vertical-align: top; +width: 16em; +border-collapse: collapse; +padding: 0; +} +/* */ +body.fullscreen #portal-column-one, +body.fullscreen #portal-column-two { +display: none; +} +body.fullscreen #portal-column-content { +width: 100%; +margin: 0; +padding: 0; +} +/* */ + +} + + +/* - authoring.css - */ +@media screen { +/* http://www.opentox.org/portal_css/authoring.css?original=1 */ +/* */ +/* */ +/* */ +.contentViews { +background-color: transparent; +padding-left: 1em; +line-height: normal; +margin: 0; +list-style: none; +border: 1px solid #5D308A; +border-top-width: 0px; +border-left-width: 0px; +border-right-width: 0px; +} +.contentViews li { +display: inline; +padding-top: 0.5em; +} +.contentViews li a { +background-color: transparent; +border: 1px solid #5D308A; +border-style: solid; +color: #333; +height: auto; +margin-right: 0.5em; +padding: 0em 1em; +line-height: normal; +text-decoration: none; +text-transform: none; +z-index: 1; +} +.contentViews .selected a { +background-color: #DDDDDD; +border-bottom: #DDDDDD 1px solid; +color: #333; +} +.contentViews li a:hover { +background-color: #DDDDDD; +color: #333; +} +.configlet .contentViews { +font-size: 90%; +} +.contentActions { +background-color: #DDDDDD; +border-left: 1px solid #5D308A; +border-right: 1px solid #5D308A; +color: #333; +text-align: right; +text-transform: none; +padding: 0 0 0 1em; +z-index: 2; +position:relative; +height: 1.6em; +} +.contentActions ul, +.contentActions li { +margin: 0; +list-style: none; +list-style-image: none; +color: #333; +text-align: left; +line-height: 1.6em; +} +.contentActions li { +float: right; +z-index: 4; +border-left: 1px solid #5D308A; +} +.contentActions a { +text-decoration: none; +color: #333; +padding: 0 0.5em; +cursor: pointer; +} +.contentActions span.subMenuTitle { +padding: 0em 0.5em; +position: relative; +white-space: nowrap; +display: inline; +} +.contentActions a span.subMenuTitle { +padding: 0px; +display: inline; +} +.actionMenu { +/* */ +position: relative; +margin: 0; +padding: 0; +} +.actionMenu .actionMenuHeader { +margin: 0; +padding: 0; +font-weight: normal; +} +.actionMenu.activated .actionMenuHeader { +position: relative; +z-index: 10; +} +.actionMenu .actionMenuHeader a { +display: block; +} +.arrowDownAlternative { +font-size: 0.85em; +} +.actionMenu .actionMenuContent { +display: none; +z-index: 5; +position: absolute; +top: 1.6em; +right: -1px; +height: auto; +padding: 0; +margin: 0; +} +.actionMenu.activated .actionMenuContent { +display: block !important; +} +.actionMenu.activated .actionMenuContent { +/* */ +display: table !important; +border-collapse: collapse; +border-spacing: 0; +} +.actionMenu.deactivated .actionMenuContent { +display: none !important; +} +.actionMenu .actionMenuContent ul { +display: block; +background: #DDDDDD; +border: 1px #5D308A; +border-style: none solid solid solid; +margin: -2px 0 0 0; +padding: 0; +} +.actionMenu .actionMenuContent li { +float: none; +background-color: transparent; +display: inline; +padding: 0; +margin: 0; +border: 0; +} +.actionMenu .actionMenuContent li a { +display: block; +white-space: nowrap; +margin: 0.2em 0; +} +.actionMenu .actionMenuContent .selected { +display: block; +white-space: nowrap; +padding: 0 0.5em; +margin: 0.2em 0; +} +.actionMenu .actionMenuContent li a:hover { +background-color: #5D308A; +color: White; +} +.actionMenu .actionMenuContent .actionSeparator a { +padding-top: 0.2em; +border-top: 1px solid #5D308A; +} +#templateMenu li a { +padding-left: 16px; +} +ul.configlets { +margin: 1em 0; +list-style-image: none; +list-style: none; +} +ul.configlets li { +margin-bottom: 1em; +} +ul.configlets li a { +text-decoration: none; +border: none; +} +ul.configlets li a:visited { +color: #5D308A; +background-color: transparent; +} +ul.configlets li a:active { +color: #5D308A; +background-color: transparent; +} +ul.configlets li label { +font-weight: bold; +} +ul.configletDetails { +margin: 0em 1em 1em 4em; +list-style-image: none; +list-style: none; +} +ul.configletDetails li { +margin-bottom: 1em; +display: inline; +} +ul.configletDetails li a { +text-decoration: none; +} +ul.configletDetails li label { +font-weight: bold; +} +ul.configletDetails li.configletDescription { +display: block; +color: #666666; +font-size: 0.9em; +margin: 0; +} +/* */ +.stx table p { +margin: 0; +padding: 0; +} +.stx table { +border: 1px solid #000 ! important; +} +.stx table td { +border-bottom: 1px solid #000; +} +.reviewHistory, +.contentHistory { +display: inline; +font-size: 110% !important; +color: Black; +} +.comment { +background: #DDDDDD; +border: 1px dashed #000; +padding: 0.25em 1em 0.5em 1em; +margin-bottom: 1em; +} +.comment h1, +.comment h2, +.comment h3, +.comment h4, +.comment h5, +.comment h6 { +border-bottom: 1px dashed #666666; +font-weight: normal; +} +.comment h3 a { +background-image: url(http://www.opentox.org/discussionitem_icon.gif); +background-repeat: no-repeat; +padding-left: 18px; +margin-left: -1px; +margin-bottom: 1px; +min-height: 1.6em; +height: auto; +line-height: 1.6em; +} +.commentBody { +margin: 0 1em 1em 1em; +} +.spacer { +margin: 1em; +} +/* */ +dl.collapsible { +border: 1px solid #000 !important; +margin: 1em 0 0 0; +padding: 0; +} +dl.collapsible dt.collapsibleHeader { +display: block; +float: left; +background: White; +line-height: 1.2em; +vertical-align: middle; +font-size: 90%; +position: relative; +top: -0.6em; +width: auto; +margin: 0 0 -0.6em 1em; +padding: 0 0.5em; +} +dl.collapsible dd.collapsibleContent { +margin: 0; +padding: 0 1em; +clear: left; +} +/* */ +dl.collapsible dd.collapsibleContent > dl { +margin: 0; +padding: 0; +} +dl.expandedInlineCollapsible dt.collapsibleHeader, +dl.expandedBlockCollapsible dt.collapsibleHeader { +padding: 0 6px 0 22px; +background: White url(treeExpanded.gif) no-repeat 6px 50%; +cursor: pointer; +} +dl.collapsedBlockCollapsible { +border: none !important; +height: 1em; +width: auto; +} +dl.collapsedBlockCollapsible dt.collapsibleHeader { +float: none; +position: static; +margin: 0; +padding: 0 0 0 22px; +line-height: 1em; +background: transparent url(treeCollapsed.gif) no-repeat 6px 50%; +cursor: pointer; +} +dl.collapsedInlineCollapsible dd.collapsibleContent, +dl.collapsedBlockCollapsible dd.collapsibleContent { +display: none; +} +dl.collapsedInlineCollapsible { +border: none !important; +height: 1em; +width: auto; +display: inline; +} +dl.collapsedInlineCollapsible dt.collapsibleHeader { +position: static; +float: none; +margin: 0; +padding: 0 0 0 22px; +line-height: 1em; +background: transparent url(treeCollapsed.gif) no-repeat 6px 50%; +cursor: pointer; +display: inline; +} +.configlet .documentEditable { +padding: 0em !important; +} +.documentEditable .documentContent { +border: 1px solid #5D308A; +padding: 0; +} +.label { +font-weight: bold; +display: inline; +padding-right: 0.5em; +} +.optionsToggle { +border: 1px solid #000; +color: #333; +background-color: #5D308A; +font-weight: normal !important; +font-size: 0.9em; +} +/* */ +.portalNotLoggedIn {} +#portal-column-content fieldset > * input:focus, +#portal-column-content fieldset > * textarea:focus { +border-color: #ffa500; +border-width: 1px; +} +/* */ +.highlightedSearchTerm { +background-color: #ffa; +} +dl.searchResults .highlightedSearchTerm { +background-color: transparent; +font-weight: bold; +} +/* */ +.noInheritedRoles { +color: #a0a0a0; +} +/* */ +.currentItem { +border-collapse: collapse; +border: 2px solid #ffa500; +padding: 1px; +} +.managePortletsLink { +display: block; +color: #666666; +font-size: 0.9em; +font-weight: normal; +} +ul.formTabs { +position: relative; +display: block; +margin: 0 0 -2em 0; +padding: 0; +list-style-type: none; +text-align: center; +} +li.formTab { +display: inline; +margin: 0; +padding: 0; +} +li.formTab a { +/* */ +display: inline-block; +} +li.formTab a { +border-top: 1px solid #000; +border-bottom: 1px solid #000; +border-left: 1px dotted #000; +background: White; +margin: 0; +padding: 0.125em 0.75em; +text-decoration: none; +} +li.formTab a:visited { +color: #5D308A; +} +li.firstFormTab a { +border-left: 1px solid #000; +} +li.lastFormTab a { +border-right: 1px solid #000; +} +li.formTab a.selected { +background: #5D308A; +} +li.formTab a:hover { +background: #5D308A; +} +li.formTab a.notify { +background-color: #ffce7b; +color: #333; +} +li.formTab a.required span { +background-image: url(http://www.opentox.org/required.gif); +background-position: center right; +background-repeat: no-repeat; +padding-right: 8px; +} +li.formTab a.notify:hover { +background-color: #ffa500; +} +.formPanel { +padding: 1em 1em 1em 1em; +border: 1px solid #000; +} +.formPanel.hidden { +display: none; +} +div.formControls input.hidden { +display: none; +} +/* */ + +} + + +/* - portlets.css - */ +@media screen { +/* http://www.opentox.org/portal_css/portlets.css?original=1 */ +/* */ +.managePortletsFallback { +margin: 0 0 0 1em; +} +/* */ +.portlet a { +text-decoration: none; +} +.portlet a.tile { +display: block; +} +/* */ +.portletItem a, +.portletFooter a { +border-bottom: none; +} +.portletItem a:visited, +.portletFooter a:visited { +color: #5D308A; +} +.portletHeader { +font-weight: normal; +line-height: 1.6em; +} +.portletItem { +margin: 0; +} +.portletItem ol { +margin: 0 0 0 1em; +} +.portletItemDetails { +text-align: right; +display: block; +color: #333; +} +.portletFooter { +background-color: #DDDDDD; +margin: 0; +text-align: right; +} +.dayPopup { +background-color: #ffffe1; +border: 1px solid Black; +padding: 0.2em; +position: absolute; +visibility: hidden; +width: 12em; +z-index: 2; +} +.date { +font-weight: bold; +} +.portletCalendar { +width: 100%; +margin: 1px 0 1em 0; +width: 100%; +} +.portletCalendar dt { +font-weight: normal; +text-align: center; +line-height: 1.6em; +border-bottom: none; +} +.portletCalendar dd { +margin: 0; +padding: 0; +} +.portletCalendar a { +text-decoration: none; +} +.portletCalendar a:hover { +text-decoration: none; +} +.ploneCalendar { +border-collapse: collapse; +border-spacing:0; +width: 100%; +} +.ploneCalendar td { +background-color: transparent; +width: 14%; +text-align: center; +padding: 2px; +} +.ploneCalendar .weekdays th { +text-align: center; +padding: 2px; +font-weight: normal; +} +.ploneCalendar .event { +font-weight: bold; +} +.ploneCalendar .todayevent { +font-weight: bold; +} +.ploneCalendar .todaynoevent { +border-collapse: collapse; +} +.managePortletsLink { +text-align: center; +} +div.portlets-manager div.section { +padding-top: 1em !important; +} +div.managedPortlet { +padding-top:0.5em; +padding-bottom:0.5em; +} +.managedPortlet .portletHeader { +min-height: 3em !important; +} +.managedPortlet a { +text-decoration: underline; +} +.managedPortletActions { +display:block; +float:right; +} +.managedPortletActions a { +text-decoration: none; +} +.managedPortletActions a.up, +.managedPortletActions a.down { +color:blue !important; +} +.managedPortletActions a.delete { +color:red !important; +} +/* */ +.toc { +float: left; +width: 30%; +font-size: 90%; +margin: 0 0 0.5em 0.5em; +} +/* */ +/* */ + +} + + +/* - controlpanel.css - */ +@media screen { +/* http://www.opentox.org/portal_css/controlpanel.css?original=1 */ +/* */ +.inlineDisplay { +display:inline +} +table.controlpanel-listing { +width:100%; +} +table.controlpanel-listing td, table.controlpanel-listing th { +font-size:120%; +} +table.controlpanel-listing dl { +margin-top:0; +} +table.controlpanel-listing dd { +margin-left: 1em; +} +table.controlpanel-listing dl dt a .trigger{ +font-weight:normal; +} +table .controlpanel-listing td { +vertical-align:top; +} +table.controlpanel-listing td.checker{ +text-align:center; +} +table.controlpanel-listing th.smallcolumn { +width:1.5em; +} +.chooser-right { +float:right; +margin-right:0 !important; +margin-bottom:0 !important; +} +.rule-element { +background-color:#EEF3F5; +margin:0.5em 0pt 0.5em; +padding:0.3em 1em 0.3em 1em; +width:auto; +} +.rule-element dl { +} +.rule-element dl dd { +margin-left:1em; +} +.rule-updown, .rule-operations { +float:right; +padding-top:0.8em; +} +/* */ + +} + + +/* - print.css - */ +@media print { +/* http://www.opentox.org/portal_css/print.css?original=1 */ +body { +font-family: Baskerville, Georgia, Garamond, Times, serif; +font-size: 11pt !important; +} +h1, h2, h3, h4, h5, h6 { +border: none; +font-family: Baskerville, Georgia, Garamond, Times, serif; +} +div, p, ul, dl, ol { +width: auto; +} +ul, ol, dl { +padding-right: 0.5em; +} +ul { +list-style-type: square; +} +.documentDescription { +font-weight: bold; +} +pre { +border: 1pt dotted black; +white-space: pre; +font-size: 8pt; +overflow: auto; +padding: 1em 0; +} +table.listing, +table.listing td { +border: 1pt solid black; +border-collapse: collapse; +} +a { +color: Black !important; +padding: 0 !important; +text-decoration: none !important; +} +a:link, a:visited { +color: #520; +background: transparent; +} +/* */ +div.pageBreak { +page-break-before: always; +} +/* */ +div.top, +#portal-logo, +#portal-siteactions, +.hiddenStructure, +#portal-searchbox, +#portal-globalnav, +#portal-personaltools, +#portal-breadcrumbs, +#portal-column-one, +#portal-column-two, +.contentViews, +.contentActions, +.help, +.legend, +.portalMessage, +.documentActions, +.documentByLine, +.netscape4, +#portal-footer, +#portal-colophon, +.skipnav, +#kss-spinner, +#review-history, +.listingBar, +.visualNoPrint { +display: none; +} +#portal-top { +display: none; +} +} + + +/* - deprecated.css - */ +@media screen { +/* http://www.opentox.org/portal_css/deprecated.css?original=1 */ +/* */ +/* */ +/* */ +/* */ +div.portalMessage, +p.portalMessage, +.system-message, +#error-handling { +background-color: #ffce7b; +border: 1px solid #ffa500; +color: #333; +font-size: 0.9em; +margin: 1em 0em; +padding: 0.5em 1em 0.5em 3em; +vertical-align: middle; +background-image: url(http://www.opentox.org/info_icon.gif); +background-repeat: no-repeat; +background-position: 5px 50%; +} +/* */ + +} + + +/* - navtree.css - */ +@media screen { +/* http://www.opentox.org/portal_css/navtree.css?original=1 */ +/* */ +/* */ +/* */ +.portletNavigationTree { +padding: 0; +list-style: none !important; +list-style-image: none !important; +line-height: 1em; +} +.navTree { +list-style: none; +list-style-image: none; +margin-top: 1px; +} +.navTree li { +margin-bottom: 1px; +} +.navTreeItem { +display: block; +padding: 0; +margin: 0; +} +.navTreeItem a, +dd.portletItem .navTreeItem a { +border: 1px solid White; +display: block; +text-decoration: none; +padding-top: 0.2em; +padding-bottom: 0.25em; +} +.navTreeItem a:hover, +dd.portletItem .navTreeItem a:hover { +background-color: #5D308A; +color: #fff; +border: 1px solid #000; +} +.navTreeCurrentItem { +background-color: #5D308A; +color: #fff; +border: 1px solid #000 !important; +} +li.navTreeCurrentItem { +display: block; +padding: 0 0 0 1px; +margin: 0 0 2px -1px; +} +li.navTreeCurrentItem a, +li.navTreeCurrentItem a:hover { +display: block; +border: 1px solid #5D308A; +min-height: 1.6em; +line-height: 1.6em; +height: auto; +} +/* */ +* html li.navTreeCurrentItem a, +* html li.navTreeCurrentItem a:hover { +height: 1.6em; +} +.navTreeLevel0 { margin: 0; } +.navTreeLevel1 { margin-left: 1em;} +.navTreeLevel2 { margin-left: 0.75em; } +.navTreeLevel3 { margin-left: 0.75em; } +.navTreeLevel4 { margin-left: 0.75em; } +.navTreeLevel5 { margin-left: 0.75em; } +/* */ + +} + + +/* - invisibles.css - */ +@media screen { +/* http://www.opentox.org/portal_css/invisibles.css?original=1 */ +/* */ +/* */ +/* */ +/* */ +/* */ +/* */ +/* */ +/* */ +/* */ +ul.visualNoMarker, +ol.visualNoMarker { +list-style-type: none; +list-style-image: none; +margin: 0.5em 0 0 0; +} +.visualOverflow { +overflow: auto; +margin: 0 0 1em 0; +} +.visualOverflow pre, +.visualOverflow table, +.visualOverflow img { +margin: 0; +} +/* */ +.hiddenStructure { +display: block; +background: transparent; +background-image: none; /* */ +border: none; +height: 1px; +overflow: hidden; +padding: 0; +margin: -1px 0 0 -1px; +width: 1px; +} +.contentViews .hiddenStructure, +.contentActions .hiddenStructure { +position: absolute; +top: -200px; +left: -200px; +} +.hiddenLabel { +display: block; +background: transparent; +background-image: none; /* */ +border: none; +height: 1px; +overflow: hidden; +padding: 0; +margin: -1px 0 0 -1px; +width: 1px; +} +/* */ +.visualClear { +display: block; +clear: both; +} +/* */ +.netscape4 { +display: none; +} +/* */ +tr.dragging td { +background-color: #ff6 !important; +} +.draggingHook { +cursor: move; +} +.notDraggable { +} +/* */ +} + + +/* - forms.css - */ +@media screen { +/* http://www.opentox.org/portal_css/forms.css?original=1 */ +/* */ +/* */ +textarea { +font: 100% Monaco, "Courier New", Courier, monospace; +border: 1px solid #ddd; +border-color:#666 #ddd #ddd #666; +color: #666666; +background: White url(http://www.opentox.org/input_background.gif) repeat-x; +width: 100%; +} +input { +font-family: Verdana,Arial,Helvetica,sans-serif; +visibility: visible; +border: 1px solid #ddd; +border-color:#666 #ddd #ddd #666; +color: #666666; +vertical-align: middle; +background: White url(http://www.opentox.org/input_background.gif) repeat-x; +font-size: 1em; +} +/* */ +.noborder, +.documentEditable * .noborder { +border: none; +margin: 0; +background: none; +background-color: transparent; +} +input[type=checkbox] { +border: none; +margin: 0; +background: none; +background-color: transparent; +} +#searchGadget { +border: 1px solid #000; +} +button { +font-family: Verdana,Arial,Helvetica,sans-serif; +visibility: visible; +border: 1px solid #000; +color: #666666; +vertical-align: middle; +background-color: #5D308A; +padding: 1px; +cursor: pointer; +font-size: 0.9em; +text-transform: none; +} +select { +vertical-align: top; +} +form { +border: none; +margin: 0; +} +fieldset { +border: 1px solid #000; +margin: 1em 0em 1em 0em; +padding: 0em 1em 1em 1em; +line-height: 1.5em; +width: auto; +} +legend { +background: White; +padding: 0.5em; +font-size: 90%; +} +label { +font-weight: bold; +} +optgroup { +font-style: normal; +font-weight: bold; +color: #999; +padding-left: 0.25em; +} +option { +color: black; +} +optgroup > option { +padding: 0 0.25em 0 1em; +} +dl.enableFormTabbing dd { +margin-left: 0; +padding-top: 2em; +} +#login-form { +width: 30em; +margin: 0 auto; +} +#login-form .field { +clear: none; +} +#login-form input { +font-size: 150%; +} +#login-form input.context { +padding: 1px 10px 1px 20px; +background-position: 9px 5px; +margin-bottom: 1em; +} +#forgotten-password { +float: right; +width: 35%; +margin: 0 1em; +} +.standalone, +.documentEditable * .standalone { +background: #5D308A url(http://www.opentox.org/linkOpaque.gif) 9px 1px no-repeat; +color: #333; +cursor: pointer; +font-size: 0.9em; +padding: 1px 1px 1px 15px; +text-transform: none; +border: 1px solid #000; +} +.context, +.formControls .actionButtons .button, +.documentEditable * .context { +background: transparent url(http://www.opentox.org/linkTransparent.gif) 9px 1px no-repeat; +color: #333; +cursor: pointer; +font-size: 0.9em; +padding: 1px 1px 1px 15px; +text-transform: none; +border: 1px solid #000; +} +.destructive, +.documentEditable * .destructive { +background: #ffce7b url(http://www.opentox.org/linkTransparent.gif) 9px 1px no-repeat; +border: 1px solid #ffa500; +color: #333; +cursor: pointer; +font-size: 0.9em; +padding: 1px 1px 1px 15px; +text-transform: none; +border: 1px solid #000; +} +input.searchButton { +margin-bottom: 1px ! important; +color: #333; +font-size: 0.9em; +background: White url(http://www.opentox.org/search_icon.gif) 2px 1px no-repeat; +cursor: pointer; +padding: 1px 1px 1px 19px; +text-transform: none; +border: 1px solid #000; +} +.searchSection { +color: #666666; +margin-top: 0.25em; +} +.searchSection label:hover { +color: #333; +} +/* */ +.field { +top: 0; +left: 0; +margin: 0 1em 1em 0; +clear: both; +} +.field .field { +margin: 0; +} +.fieldRequired { +background: url(http://www.opentox.org/required.gif) center left no-repeat; +padding: 0 0 0 8px; +color: White; +} +.fieldUploadFile { +text-align: right; +margin-right: 0.75em; +display: none; +} +.fieldTextFormat { +text-align: right; +margin-right: 0.75em +} +.formHelp { +font-size: 90%; +color: #666666; +margin: 0 0 0.2em 0; +} +.formHelp:hover { +color: #333; +cursor: default; +} +div.error { +/* */ +background-color: #fdc; +border: 1px solid #d00; +padding: 0.5em; +margin: 0 0 1em 0; +width: 95%; +} +.error .fieldRequired { +color: #d00; +} +/* */ +#archetypes-fieldname-title input, input#form\.title { +font-size: 160%; +font-family: Arial,Helvetica,Verdana,Geneva,sans-serif; +font-weight: normal; +width: 99%; +} +#archetypes-fieldname-description textarea, textarea#form\.description { +font: 100% Verdana,Arial,Helvetica,sans-serif; +font-weight: bold; +} +input.inputLabelActive { +color: #666666; +} +textarea#form\.description { +height: 6em; +width: 99%; +} +tr.selected { +background-color: #ffa; +} +.kupu-save-message { +color: white; +font-weight: bold; +background-color: red; +padding: 0.3em; +position: fixed; +top: 0; +right: 0; +z-index: 999; +} +/* */ + +} + + +/* - ploneKss.css - */ +@media screen { +/* http://www.opentox.org/portal_css/ploneKss.css?original=1 */ +#kss-spinner { +display:none; +width: 20px; +height: 20px; +position: fixed; +background-position: center center; +top: 50%; +left: 50%; +margin-top: -10px; +margin-left: -10px; +} +.formlibInlineEditable, +.inlineEditable { +padding: 1px; +display: block; +} +body.kssActive .inlineEditable:hover, +body.kssActive .formlibInlineEditable:hover { +padding: 0; +border: 1px solid #ddd; +border-color: #666 #ddd #ddd #666; +cursor: text; +background: White url(input_background.gif) repeat-x; +} +body.kssActive .inlineEditable a:hover, +body.kssActive .formlibInlineEditable a:hover { +cursor: pointer; +} +body.kssActive .formlibInlineForm:hover, +body.kssActive .formlibInlineForm a:hover { +padding: 1px; +border: none; +cursor: default; +background: none; +} +.inlineForm .formHelp, +.inlineForm .fieldRequired, +.inlineForm label { +display: none; +} +/* */ +.inlineForm .ArchetypesSelectionWidget label { +display: inline; +} +.inlineForm #archetypes-fieldname-title input, +.inlineForm #archetypes-fieldname-title { +font-size: 100%; +} +/* */ +h1 div.formControls input { +font-size: 55% !important; +} +} + -- cgit v1.2.3 From a5b5c0790de08b4deee7d72851070201cca1c744 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 17 Jun 2011 12:11:02 +0200 Subject: prefer prediction_feature, use Model.dependentVariables if no prediction feature is specified --- validation/validation_service.rb | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 8d64cd1..77cc694 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -135,7 +135,6 @@ module Validation params[alg_param[0].to_sym] = alg_param[1] end end - LOGGER.debug "building model '"+algorithm_uri.to_s+"' "+params.inspect algorithm = OpenTox::Algorithm::Generic.new(algorithm_uri) params[:subjectid] = subjectid @@ -169,26 +168,12 @@ module Validation model = OpenTox::Model::Generic.find(self.model_uri, self.subjectid) unless self.algorithm_uri -# self.attributes = { :algorithm_uri => model.algorithm } -# self.save! - #self.update :algorithm_uri => model.algorithm self.algorithm_uri = model.metadata[OT.algorithm] end - - if self.prediction_feature and model.uri=~/ambit2\/model/ - LOGGER.warn "REMOVE AMBIT HACK TO __NOT__ RELY ON DEPENDENT VARIABLE" - else + if self.prediction_feature.to_s.size==0 dependentVariables = model.metadata[OT.dependentVariables] - if self.prediction_feature - raise OpenTox::NotFoundError.new "error validating model: model.dependent_variable != validation.prediction_feature ("+ - dependentVariables.to_s+" != "+self.prediction_feature+"), model-metadata is "+model.metadata.inspect if self.prediction_feature!=dependentVariables - else - raise OpenTox::NotFoundError.new "model has no dependentVariables specified, please give prediction feature for model validation" unless dependentVariables - #self.attributes = { :prediction_feature => model.dependentVariables } - #self.save! - #self.update :prediction_feature => model.dependentVariables - self.prediction_feature = model.metadata[OT.dependentVariables] - end + raise OpenTox::NotFoundError.new "model has no dependentVariables specified, please give prediction_feature for model validation" unless dependentVariables + self.prediction_feature = model.metadata[OT.dependentVariables] end prediction_dataset_uri = "" -- cgit v1.2.3 From f537002fc16c14f8fe66b92aaebdb263aa6be6a2 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 17 Jun 2011 15:54:20 +0200 Subject: do not empty subjectid to jnlp when a&A is disabled --- reach_reports/reach_application.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/reach_reports/reach_application.rb b/reach_reports/reach_application.rb index c0e89b5..e35df7b 100755 --- a/reach_reports/reach_application.rb +++ b/reach_reports/reach_application.rb @@ -235,12 +235,14 @@ EOF jnlp.chomp! jnlp += File.join(url_for("/reach_report/QMRF",:full),params[:id]) - jnlp += <0 + jnlp += < --subjectid= EOF - jnlp.chomp! - jnlp += @subjectid.to_s + jnlp.chomp! + jnlp += @subjectid.to_s + end jnlp += < -- cgit v1.2.3 From c87b0d6866acb246d35fc457805340aef28147b8 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 17 Jun 2011 16:19:11 +0200 Subject: fix chart keys (removing 'default', adding crossvalidation fold) --- report/plot_factory.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/report/plot_factory.rb b/report/plot_factory.rb index 78d2e05..d6c7d35 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -106,6 +106,7 @@ module Reports attribute_values.each do |value| begin data << transform_roc_predictions(validation_set.filter({split_set_attribute => value}), class_value, false ) + data[-1].name = split_set_attribute.to_s.nice_attr+" "+value.to_s rescue LOGGER.warn "could not create ROC plot for "+value.to_s end @@ -133,7 +134,7 @@ module Reports attribute_values.each do |value| begin data = transform_confidence_predictions(validation_set.filter({split_set_attribute => value}), class_value, false) - names << value.to_s + names << split_set_attribute.to_s.nice_attr+" "+value.to_s confidence << data[:confidence][0] performance << data[:performance][0] rescue @@ -298,7 +299,7 @@ module Reports tp_fp_rates[:youden].each do |point,confidence| labels << ["confidence: "+confidence.to_nice_s, point[0], point[1]] end if add_label - RubyPlot::LinePlotData.new(:name => "default", :x_values => tp_fp_rates[:fp_rate], :y_values => tp_fp_rates[:tp_rate], :labels => labels) + RubyPlot::LinePlotData.new(:name => "", :x_values => tp_fp_rates[:fp_rate], :y_values => tp_fp_rates[:tp_rate], :labels => labels) end @@ -337,7 +338,7 @@ module Reports else confidence_values = validation_set.validations[0].get_predictions.get_prediction_values(class_value) pref_conf_rates = get_performance_confidence_rates(confidence_values, validation_set.unique_feature_type) - return { :names => ["default"], :performance => [pref_conf_rates[:performance]], :confidence => [pref_conf_rates[:confidence]] } + return { :names => [""], :performance => [pref_conf_rates[:performance]], :confidence => [pref_conf_rates[:confidence]] } end end -- cgit v1.2.3 From c945cad0eb0b01fb897b0dbb95d14a9872da5e1c Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 29 Jun 2011 11:19:02 +0200 Subject: do delete policies in background with sleeps, fix cleanup method --- validation/validation_application.rb | 28 ++++++++--------- validation/validation_service.rb | 60 ++++++++++++++++++++++-------------- 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index bd55d4c..c9a5649 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -61,17 +61,14 @@ post '/crossvalidation/cleanup/?' do LOGGER.info "crossvalidation cleanup, starting..." content_type "text/uri-list" deleted = [] - #Validation::Crossvalidation.find_like(params).each do |cv| - Validation::Crossvalidation.all( { :finished => false } ).each do |cv| - #num_vals = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv.id } ).size - #num_vals = Validation::Validation.all( :crossvalidation_id => cv.id ).size - #if cv.num_folds != num_vals || !cv.finished + Validation::Crossvalidation.all.collect.delete_if{|cv| cv.finished}.each do |cv| + if OpenTox::Authorization.authorized?(cv.crossvalidation_uri,"DELETE",@subjectid) LOGGER.debug "delete cv with id:"+cv.id.to_s+", finished is false" deleted << cv.crossvalidation_uri - #Validation::Crossvalidation.delete(cv.id) cv.subjectid = @subjectid cv.delete_crossvalidation - #end + sleep 1 if AA_SERVER + end end LOGGER.info "crossvalidation cleanup, deleted "+deleted.size.to_s+" cvs" deleted.join("\n")+"\n" @@ -189,8 +186,8 @@ delete '/crossvalidation/:id/?' do # Validation::Crossvalidation.delete(params[:id]) cv = Validation::Crossvalidation.get(params[:id]) - cv.subjectid = @subjectid raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found." unless cv + cv.subjectid = @subjectid cv.delete_crossvalidation end @@ -474,13 +471,14 @@ post '/cleanup/?' do LOGGER.info "validation cleanup, starting..." content_type "text/uri-list" deleted = [] - #Validation::Validation.find( :all, :conditions => { :prediction_dataset_uri => nil } ).each do |val| - Validation::Validation.all( :finished => false ).each do |val| - LOGGER.debug "delete val with id:"+val.id.to_s+", finished is false" - deleted << val.validation_uri - #Validation::Validation.delete(val.id) - val.subjectid = @subjectid - val.delete_validation + Validation::Validation.all.collect.delete_if{|val| val.finished}.each do |val| + if OpenTox::Authorization.authorized?(val.validation_uri,"DELETE",@subjectid) + LOGGER.debug "delete val with id:"+val.id.to_s+", finished is false" + deleted << val.validation_uri + val.subjectid = @subjectid + val.delete_validation + sleep 1 if AA_SERVER + end end LOGGER.info "validation cleanup, deleted "+deleted.size.to_s+" validations" deleted.join("\n")+"\n" diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 77cc694..8dc90e2 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -84,38 +84,45 @@ module Validation if (delete_all) to_delete = [:model_uri, :training_dataset_uri, :test_dataset_uri, :test_target_dataset_uri, :prediction_dataset_uri ] case self.validation_type - when /test_set_validation/ + when "test_set_validation" to_delete -= [ :model_uri, :training_dataset_uri, :test_dataset_uri, :test_target_dataset_uri ] - when /bootstrapping/ + when "bootstrapping" to_delete -= [ :test_target_dataset_uri ] - when /training_test_validation/ + when "training_test_validation" to_delete -= [ :training_dataset_uri, :test_dataset_uri, :test_target_dataset_uri ] - when /training_test_split/ + when "training_test_split" to_delete -= [ :test_target_dataset_uri ] - when /validate_dataset/ + when "validate_datasets" to_delete = [] - when /crossvalidation/ + when "crossvalidation" to_delete -= [ :test_target_dataset_uri ] + when "crossvalidation_statistics" + to_delete = [] else - raise "unknown dataset type" + raise "unknown validation type '"+self.validation_type.to_s+"'" end - to_delete.each do |attr| - uri = self.send(attr) - LOGGER.debug "also deleting "+attr.to_s+" : "+uri.to_s if uri - begin - OpenTox::RestClientWrapper.delete(uri, :subjectid => subjectid) if uri - rescue => ex - LOGGER.warn "could not delete "+uri.to_s+" : "+ex.message.to_s + Thread.new do # do deleting in background to not cause a timeout + to_delete.each do |attr| + uri = self.send(attr) + LOGGER.debug "also deleting "+attr.to_s+" : "+uri.to_s if uri + begin + OpenTox::RestClientWrapper.delete(uri, :subjectid => subjectid) if uri + sleep 1 if AA_SERVER # wait a second to not stress the a&a service too much + rescue => ex + LOGGER.warn "could not delete "+uri.to_s+" : "+ex.message.to_s + end end end end self.delete if (subjectid) - begin - res = OpenTox::Authorization.delete_policies_from_uri(validation_uri, subjectid) - LOGGER.debug "Deleted validation policy: #{res}" - rescue - LOGGER.warn "Policy delete error for validation: #{validation_uri}" + Thread.new do + begin + res = OpenTox::Authorization.delete_policies_from_uri(validation_uri, subjectid) + LOGGER.debug "Deleted validation policy: #{res}" + rescue + LOGGER.warn "Policy delete error for validation: #{validation_uri}" + end end end "Successfully deleted validation "+self.id.to_s+"." @@ -272,12 +279,18 @@ module Validation # deletes a crossvalidation, all validations are deleted as well def delete_crossvalidation - Validation.find(:crossvalidation_id => self.id).each do |v| + validations = Validation.find(:crossvalidation_id => self.id) + Thread.new do # do deleting in background to not cause a timeout + validations.each do |v| v.subjectid = self.subjectid + LOGGER.debug "deleting cv-validation "+v.validation_uri.to_s v.delete_validation + sleep 1 if AA_SERVER # wait a second to not stress the a&a service too much end - self.delete - if (subjectid) + end + self.delete + if (subjectid) + Thread.new do begin res = OpenTox::Authorization.delete_policies_from_uri(crossvalidation_uri, subjectid) LOGGER.debug "Deleted crossvalidation policy: #{res}" @@ -285,7 +298,8 @@ module Validation LOGGER.warn "Policy delete error for crossvalidation: #{crossvalidation_uri}" end end - "Successfully deleted crossvalidation "+self.id.to_s+"." + end + "Successfully deleted crossvalidation "+self.id.to_s+"." end # creates the cv folds -- cgit v1.2.3 From 0b0d101c77c2f40769036fb1455e8143f4dace9c Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 29 Jun 2011 11:56:31 +0200 Subject: add dataset cleanup method --- validation/validation_application.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index c9a5649..1c466cc 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -484,6 +484,31 @@ post '/cleanup/?' do deleted.join("\n")+"\n" end +post '/cleanup_datasets/?' do + LOGGER.info "dataset cleanup, starting..." + content_type "text/uri-list" + used_datasets = Set.new + Validation::Crossvalidation.all.each do |cv| + used_datasets << cv.dataset_uri + end + Validation::Validation.all.each do |val| + used_datasets << val.training_dataset_uri + used_datasets << val.test_target_dataset_uri + used_datasets << val.test_dataset_uri + used_datasets << val.prediction_dataset_uri + end + deleted = [] + OpenTox::Dataset.all.each do |d| + if !used_datasets.include?(d.uri) and OpenTox::Authorization.authorized?(d.uri,"DELETE",@subjectid) + deleted << d.uri + d.delete(@subjectid) + sleep 1 if AA_SERVER + end + end + LOGGER.info "dataset cleanup, deleted "+deleted.size.to_s+" datasets" + deleted.join("\n")+"\n" +end + post '/plain_training_test_split' do LOGGER.info "creating pure training test split "+params.inspect raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] -- cgit v1.2.3 From 95de1906994672c12d4d4b6fd1f5f09695fb1c60 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 30 Jun 2011 23:49:22 +0200 Subject: adding confidence-weighted regression stats --- lib/predictions.rb | 43 ++++++++++++++++++++++++++++++++++++++++++- lib/validation_db.rb | 5 +++-- report/report_factory.rb | 8 +++++--- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/lib/predictions.rb b/lib/predictions.rb index 2409375..4e34c90 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -78,6 +78,8 @@ module Lib @num_predicted = 0 @num_unpredicted = 0 + @mean_confidence = 0 + case @feature_type when "classification" @@ -111,6 +113,9 @@ module Lib @sum_multiply = 0 @sum_squares_actual = 0 @sum_squares_predicted = 0 + + @sum_weighted_abs_error = 0 + @sum_weighted_squared_error = 0 end end @@ -127,6 +132,7 @@ module Lib @num_predicted += 1 @conf_provided |= confidence_value!=nil + @mean_confidence = (confidence_value + @mean_confidence*(@num_predicted-1)) / @num_predicted.to_f if @conf_provided case @feature_type when "classification" @@ -140,7 +146,9 @@ module Lib delta = predicted_value - actual_value @sum_error += delta @sum_abs_error += delta.abs + @sum_weighted_abs_error += delta.abs*confidence_value @sum_squared_error += delta**2 + @sum_weighted_squared_error += (delta**2)*confidence_value old_prediction_mean = @prediction_mean @prediction_mean = (@prediction_mean * (@num_predicted-1) + predicted_value) / @num_predicted.to_f @@ -466,11 +474,23 @@ module Lib Math.sqrt(@sum_squared_error / (@num_with_actual_value - @num_unpredicted).to_f) end + def weighted_root_mean_squared_error + return 0 unless confidence_values_available? + return 0 if (@num_with_actual_value - @num_unpredicted)==0 + Math.sqrt(@sum_weighted_squared_error / ((@num_with_actual_value - @num_unpredicted).to_f * @mean_confidence )) + end + def mean_absolute_error return 0 if (@num_with_actual_value - @num_unpredicted)==0 @sum_abs_error / (@num_with_actual_value - @num_unpredicted).to_f end + def weighted_mean_absolute_error + return 0 unless confidence_values_available? + return 0 if (@num_with_actual_value - @num_unpredicted)==0 + @sum_weighted_abs_error / ((@num_with_actual_value - @num_unpredicted).to_f * @mean_confidence ) + end + def sum_squared_error return @sum_squared_error end @@ -486,6 +506,14 @@ module Lib ( r_2.infinite? || r_2.nan? ) ? 0 : r_2 end + def weighted_r_square + return 0 unless confidence_values_available? + ss_tot = weighted_total_sum_of_squares + return 0 if ss_tot==0 + r_2 = 1 - weighted_residual_sum_of_squares / ss_tot + ( r_2.infinite? || r_2.nan? ) ? 0 : r_2 + end + def sample_correlation_coefficient # formula see http://en.wikipedia.org/wiki/Correlation_and_dependence#Pearson.27s_product-moment_coefficient scc = ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) / @@ -498,7 +526,16 @@ module Lib #return @variance_actual * ( @num_predicted - 1 ) sum = 0 @predicted_values.size.times do |i| - sum += (@actual_values[i]-@actual_mean)**2 if @predicted_values[i]!=nil + sum += (@actual_values[i]-@actual_mean)**2 if @actual_values[i]!=nil and @predicted_values[i]!=nil + end + sum + end + + def weighted_total_sum_of_squares + return 0 unless confidence_values_available? + sum = 0 + @predicted_values.size.times do |i| + sum += ((@actual_values[i]-@actual_mean)**2)*@confidence_values[i] if @actual_values[i]!=nil and @predicted_values[i]!=nil end sum end @@ -507,6 +544,10 @@ module Lib sum_squared_error end + def weighted_residual_sum_of_squares + @sum_weighted_squared_error + end + def target_variance_predicted return @variance_predicted end diff --git a/lib/validation_db.rb b/lib/validation_db.rb index c38b82e..fb7a8b5 100755 --- a/lib/validation_db.rb +++ b/lib/validation_db.rb @@ -35,8 +35,9 @@ module Validation VAL_CLASS_PROPS = VAL_CLASS_PROPS_SINGLE + VAL_CLASS_PROPS_PER_CLASS # :regression_statistics - VAL_REGR_PROPS = [ :root_mean_squared_error, :mean_absolute_error, :r_square, - :target_variance_actual, :target_variance_predicted, :sum_squared_error, :sample_correlation_coefficient ] + VAL_REGR_PROPS = [ :root_mean_squared_error, :mean_absolute_error, :r_square, :weighted_r_square, + :target_variance_actual, :target_variance_predicted, :sum_squared_error, :sample_correlation_coefficient, + :weighted_mean_absolute_error, :weighted_root_mean_squared_error ] CROSS_VAL_PROPS = [:dataset_uri, :num_folds, :stratified, :random_seed] CROSS_VAL_PROPS_REDUNDANT = [:crossvalidation_uri, :algorithm_uri, :date] + CROSS_VAL_PROPS diff --git a/report/report_factory.rb b/report/report_factory.rb index d16066e..fcd9bab 100755 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -7,10 +7,12 @@ VAL_ATTR_CV = [ :algorithm_uri, :dataset_uri, :num_folds, :crossvalidation_fold # selected attributes of interest when performing classification VAL_ATTR_CLASS = [ :num_instances, :num_unpredicted, :accuracy, :weighted_accuracy, :weighted_area_under_roc, :area_under_roc, :f_measure, :true_positive_rate, :true_negative_rate ] -VAL_ATTR_REGR = [ :num_instances, :num_unpredicted, :root_mean_squared_error, :mean_absolute_error, :r_square ] +VAL_ATTR_REGR = [ :num_instances, :num_unpredicted, :root_mean_squared_error, + :weighted_root_mean_squared_error, :mean_absolute_error, :weighted_mean_absolute_error, :r_square, :weighted_r_square ] -VAL_ATTR_BAR_PLOT_CLASS = [ :accuracy, :weighted_area_under_roc, - :area_under_roc, :f_measure, :true_positive_rate, :true_negative_rate ] +#VAL_ATTR_BAR_PLOT_CLASS = [ :accuracy, :weighted_area_under_roc, +# :area_under_roc, :f_measure, :true_positive_rate, :true_negative_rate ] +VAL_ATTR_BAR_PLOT_CLASS = [ :accuracy, :f_measure, :true_positive_rate, :true_negative_rate ] VAL_ATTR_BAR_PLOT_REGR = [ :root_mean_squared_error, :mean_absolute_error, :r_square ] -- cgit v1.2.3 From 2ae0e7a344954b97b66c47094292084741b320e4 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 1 Jul 2011 11:18:41 +0200 Subject: add control of t-tests to algorihtm-comparison reports --- report/report_application.rb | 10 +++++++--- report/report_content.rb | 37 ++++++++++++++++++++++--------------- report/report_factory.rb | 27 +++++++++++++++++++-------- report/report_service.rb | 5 +++-- report/statistical_test.rb | 10 +++++----- report/validation_data.rb | 27 +++++++++++++++++++-------- 6 files changed, 75 insertions(+), 41 deletions(-) diff --git a/report/report_application.rb b/report/report_application.rb index 8f74bd9..b96fb27 100755 --- a/report/report_application.rb +++ b/report/report_application.rb @@ -68,8 +68,12 @@ get '/report/:report_type' do # if searching for a report, ?validation="uri" or ?crossvalidaiton="uri" is given as search param # use this (search param has equal name as report type) as default value for validation_uri post_command.attributes << OpenTox::PostAttribute.new("validation_uris",true,params[params[:report_type]],val_uri_description) - post_command.attributes << OpenTox::PostAttribute.new("identifier",true,nil,"Specifiy one identifier for each uri, separated with ','") if - params[:report_type]=="algorithm_comparison" + if params[:report_type]=="algorithm_comparison" + post_command.attributes << OpenTox::PostAttribute.new("identifier",true,nil,"Specifiy one identifier for each uri, separated with ','") + post_command.attributes << OpenTox::PostAttribute.new("ttest_significance",false,"0.9","Significance level for t-tests (Set to '0' to disable t-test).") + post_command.attributes << OpenTox::PostAttribute.new("ttest_attributes",false,nil,"Attributes for t-test; default for classification: '"+ + VAL_ATTR_TTEST_CLASS.join(",")+"', default for regression: '"+VAL_ATTR_TTEST_REGR.join(",")+"'") + end content_type "text/html" OpenTox.text_to_html rs.get_all_reports(params[:report_type], params),@subjectid,related_links,description,post_command else @@ -129,7 +133,7 @@ post '/report/:type' do task = OpenTox::Task.create("Create report",url_for("/report/"+params[:type], :full)) do |task| #,params perform do |rs| rs.create_report(params[:type],params[:validation_uris]?params[:validation_uris].split(/\n|,/):nil, - params[:identifier]?params[:identifier].split(/\n|,/):nil,@subjectid,task) + params[:identifier]?params[:identifier].split(/\n|,/):nil,params,@subjectid,task) end end return_task(task) diff --git a/report/report_content.rb b/report/report_content.rb index 30118cf..ea2ad5a 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -25,28 +25,35 @@ class Reports::ReportContent def add_paired_ttest_tables( validation_set, group_attribute, test_attributes, + ttest_level = 0.9, section_title = "Paired t-test", section_text = nil) + raise "no test_attributes given: "+test_attributes.inspect unless test_attributes.is_a?(Array) and test_attributes.size>0 section_test = @xml_report.add_section(@current_section, section_title) @xml_report.add_paragraph(section_test, section_text) if section_text + + test_attributes.each do |test_attribute| + accept_values = validation_set.get_accept_values_for_attr(test_attribute) + accept_values = [nil] unless accept_values and accept_values.size>0 + #puts "t-test for "+test_attribute.to_s+", class values: "+accept_values.to_s - test_attributes.each do |test_attribute| - level = 0.90 - test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, - group_attribute, test_attribute, "paired_ttest", level ) - #puts test_matrix.inspect - titles = test_matrix[:titles] - matrix = test_matrix[:matrix] - table = [] - #puts titles.inspect - table << [""] + titles - titles.size.times do |i| - table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") } + accept_values.each do |accept_value| + test_matrix = Reports::ReportStatisticalTest.test_matrix( validation_set.validations, + group_attribute, test_attribute, accept_value, "paired_ttest", ttest_level ) + #puts test_matrix.inspect + titles = test_matrix[:titles] + matrix = test_matrix[:matrix] + table = [] + #puts titles.inspect + table << [""] + titles + titles.size.times do |i| + table << [titles[i]] + matrix[i].collect{|v| (v==nil || v==0) ? "" : (v<0 ? "-" : "+") } + end + accept_value_str = accept_value!=nil ? " for class-value '"+accept_value.to_s+"'" : "" + @xml_report.add_table(section_test, test_attribute.to_s+accept_value_str+", significance-level: "+ttest_level.to_s+", num results: "+ + test_matrix[:num_results].to_s, table, true, true) end - - @xml_report.add_table(section_test, test_attribute.to_s+", significance-level: "+level.to_s+", num results: "+ - test_matrix[:num_results].to_s, table, true, true) end Reports::ReportStatisticalTest.quit_r end diff --git a/report/report_factory.rb b/report/report_factory.rb index fcd9bab..25bc4d9 100755 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -15,6 +15,9 @@ VAL_ATTR_REGR = [ :num_instances, :num_unpredicted, :root_mean_squared_error, VAL_ATTR_BAR_PLOT_CLASS = [ :accuracy, :f_measure, :true_positive_rate, :true_negative_rate ] VAL_ATTR_BAR_PLOT_REGR = [ :root_mean_squared_error, :mean_absolute_error, :r_square ] +VAL_ATTR_TTEST_REGR = [:r_square, :root_mean_squared_error] +VAL_ATTR_TTEST_CLASS = [:percent_correct, :weighted_area_under_roc] + # = Reports::ReportFactory # @@ -33,14 +36,14 @@ module Reports::ReportFactory # call-seq: # self.create_report(type, validation_set) => Reports::ReportContent # - def self.create_report(type, validation_set, task=nil) + def self.create_report(type, validation_set, params={}, task=nil) case type when RT_VALIDATION create_report_validation(validation_set, task) when RT_CV create_report_crossvalidation(validation_set, task) when RT_ALG_COMP - create_report_compare_algorithms(validation_set, task) + create_report_compare_algorithms(validation_set, params, task) else raise "unknown report type "+type.to_s end @@ -151,7 +154,7 @@ module Reports::ReportFactory report end - def self.create_report_compare_algorithms(validation_set, task=nil) + def self.create_report_compare_algorithms(validation_set, params={}, task=nil) #validation_set.to_array([:test_dataset_uri, :model_uri, :algorithm_uri], false).each{|a| puts a.inspect} raise OpenTox::BadRequestError.new("num validations is not >1") unless validation_set.size>1 @@ -165,13 +168,13 @@ module Reports::ReportFactory else raise OpenTox::BadRequestError.new("num different cross-validation-ids <2") if validation_set.num_different_values(:crossvalidation_id)<2 validation_set.load_cv_attributes - compare_algorithms_crossvalidation(validation_set, task) + compare_algorithms_crossvalidation(validation_set, params, task) end end # create Algorithm Comparison report # crossvalidations, 1-n datasets, 2-n algorithms - def self.compare_algorithms_crossvalidation(validation_set, task=nil) + def self.compare_algorithms_crossvalidation(validation_set, params={}, task=nil) # groups results into sets with equal dataset if (validation_set.num_different_values(:dataset_uri)>1) @@ -203,13 +206,21 @@ module Reports::ReportFactory case validation_set.unique_feature_type when "classification" result_attributes += VAL_ATTR_CLASS - ttest_attributes = [:percent_correct, :weighted_area_under_roc] + ttest_attributes = VAL_ATTR_TTEST_CLASS bar_plot_attributes = VAL_ATTR_BAR_PLOT_CLASS else result_attributes += VAL_ATTR_REGR - ttest_attributes = [:r_square, :root_mean_squared_error] + ttest_attributes = VAL_ATTR_TTEST_REGR bar_plot_attributes = VAL_ATTR_BAR_PLOT_REGR end + + if params[:ttest_attributes] and params[:ttest_attributes].chomp.size>0 + ttest_attributes = params[:ttest_attributes].split(",").collect{|a| a.to_sym} + end + ttest_significance = 0.9 + if params[:ttest_significance] + ttest_significance = params[:ttest_significance].to_f + end dataset_grouping.each do |validations| @@ -230,7 +241,7 @@ module Reports::ReportFactory report.add_result(merged,result_attributes,res_titel,res_titel,res_text) # pending: regression stats have different scales!!! report.add_bar_plot(merged, :identifier, bar_plot_attributes) if validation_set.unique_feature_type=="classification" - report.add_paired_ttest_tables(set, :identifier, ttest_attributes) + report.add_paired_ttest_tables(set, :identifier, ttest_attributes, ttest_significance) if ttest_significance>0 report.end_section end task.progress(100) if task diff --git a/report/report_service.rb b/report/report_service.rb index 3e23889..f299122 100644 --- a/report/report_service.rb +++ b/report/report_service.rb @@ -60,8 +60,9 @@ module Reports # call-seq: # create_report(type, validation_uris) => string # - def create_report(type, validation_uris, identifier=nil, subjectid=nil, task=nil) + def create_report(type, validation_uris, identifier=nil, params={}, subjectid=nil, task=nil) + raise "params is no hash" unless params.is_a?(Hash) LOGGER.info "create report of type '"+type.to_s+"'" check_report_type(type) @@ -77,7 +78,7 @@ module Reports task.progress(10) if task #step 2: create report of type - report_content = Reports::ReportFactory.create_report(type, validation_set, + report_content = Reports::ReportFactory.create_report(type, validation_set, params, OpenTox::SubTask.create(task,10,90)) LOGGER.debug "report created" diff --git a/report/statistical_test.rb b/report/statistical_test.rb index 9461cac..8d6bd62 100644 --- a/report/statistical_test.rb +++ b/report/statistical_test.rb @@ -38,7 +38,7 @@ module Reports class ReportStatisticalTest # __grouped_validations__ : array of validation arrays - def self.test_matrix( validations, group_attribute, test_attribute, test_method="paired_ttest", significance_level=0.95 ) + def self.test_matrix( validations, group_attribute, test_attribute, class_value, test_method="paired_ttest", significance_level=0.95 ) raise "statistical-test: '"+test_method+"' does not exist" unless ReportStatisticalTest.respond_to?(test_method) grouped_validations = Reports::Util.group(validations, [group_attribute]) @@ -60,17 +60,17 @@ module Reports validations2 = grouped_validations[j] title2 = validations2[0].send(group_attribute) matrix[i][j] = ReportStatisticalTest.send(test_method,validations1,validations2, - test_attribute, significance_level) + test_attribute, class_value, significance_level) end end end {:titles => titles, :matrix => matrix, :num_results => grouped_validations[0].size} end - def self.paired_ttest( validations1, validations2, attribute, significance_level=0.95 ) + def self.paired_ttest( validations1, validations2, attribute, class_value, significance_level=0.95 ) - array1 = validations1.collect{ |v| v.send(attribute) } - array2 = validations2.collect{ |v| v.send(attribute) } + array1 = validations1.collect{ |v| (v.send(attribute).is_a?(Hash) ? v.send(attribute)[class_value] : v.send(attribute)) } + array2 = validations2.collect{ |v| (v.send(attribute).is_a?(Hash) ? v.send(attribute)[class_value] : v.send(attribute)) } LOGGER.debug "paired-t-testing "+attribute.to_s+" "+array1.inspect+" vs "+array2.inspect LIB::StatisticalTest.pairedTTest(array1, array2, significance_level) end diff --git a/report/validation_data.rb b/report/validation_data.rb index fa0af8e..aa146a6 100755 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -1,8 +1,9 @@ # the variance is computed when merging results for these attributes VAL_ATTR_VARIANCE = [ :area_under_roc, :percent_correct, :root_mean_squared_error, :mean_absolute_error, - :r_square, :accuracy, :weighted_area_under_roc, :weighted_accuracy ] -VAL_ATTR_RANKING = [ :area_under_roc, :percent_correct, :true_positive_rate, :true_negative_rate, :weighted_area_under_roc ] #:accuracy ] + :r_square, :accuracy, :weighted_area_under_roc, :weighted_accuracy, :weighted_root_mean_squared_error, :weighted_mean_absolute_error, + :weighted_r_square ] +VAL_ATTR_RANKING = [ :area_under_roc, :percent_correct, :true_positive_rate, :true_negative_rate, :weighted_area_under_roc, :accuracy, :f_measure ] ATTR_NICE_NAME = {} @@ -24,7 +25,7 @@ class Object if self==0 return "0" elsif abs>0.1 - return "%.2f" % self + return "%.3f" % self elsif abs>0.01 return "%.3f" % self else @@ -310,9 +311,9 @@ module Reports def to_table( attribute_col, attribute_row, attribute_val) row_values = get_values(attribute_row) - #puts row_values.inspect + #puts "row: "+row_values.inspect col_values = get_values(attribute_col) - #puts col_values.inspect + #puts "col: "+col_values.inspect # get domain for classification attribute, i.e. ["true","false"] accept_values = get_accept_values_for_attr(attribute_val) @@ -326,7 +327,7 @@ module Reports val = nil @validations.each do |v| if v.send(attribute_row)==row and v.send(attribute_col)==col - raise "two validation have equal row and column values"if val!=nil + #raise "two validation have equal row and column values: "+val.to_s if val!=nil val = v.send(attribute_val) val = val[accept_values[0]] if first_value_elem val = val.to_nice_s @@ -449,8 +450,18 @@ module Reports return new_set end - def sort(attribute, ascending=true) - @validations.sort!{ |a,b| a.send(attribute).to_s <=> b.send(attribute).to_s } + def sort(attributes, ascending=true) + attributes = [attributes] unless attributes.is_a?(Array) + @validations.sort! do |a,b| + val = 0 + attributes.each do |attr| + if a.send(attr).to_s != b.send(attr).to_s + val = a.send(attr).to_s <=> b.send(attr).to_s + break + end + end + val + end end # creates a new validaiton set, that contains a ranking for __ranking_attribute__ -- cgit v1.2.3 From d51a6d01d2a54512f1969e3ef98f5beb24c643e9 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 1 Jul 2011 11:35:59 +0200 Subject: small bugifx in building error msg --- report/report_factory.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/report/report_factory.rb b/report/report_factory.rb index 25bc4d9..7ec2e1c 100755 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -107,7 +107,7 @@ module Reports::ReportFactory validation_set.unique_value(:num_folds).to_s+")") unless validation_set.unique_value(:num_folds).to_i==validation_set.size raise OpenTox::BadRequestError.new("num different folds is not equal to num validations") unless validation_set.num_different_values(:crossvalidation_fold)==validation_set.size raise OpenTox::BadRequestError.new("validations must have unique feature type, i.e. must be either all regression, "+ - +"or all classification validations") unless validation_set.unique_feature_type + "or all classification validations") unless validation_set.unique_feature_type pre_load_predictions( validation_set, OpenTox::SubTask.create(task,0,80) ) validation_set.validations.sort! do |x,y| x.crossvalidation_fold.to_f <=> y.crossvalidation_fold.to_f @@ -159,7 +159,7 @@ module Reports::ReportFactory #validation_set.to_array([:test_dataset_uri, :model_uri, :algorithm_uri], false).each{|a| puts a.inspect} raise OpenTox::BadRequestError.new("num validations is not >1") unless validation_set.size>1 raise OpenTox::BadRequestError.new("validations must have unique feature type, i.e. must be either all regression, "+ - +"or all classification validations") unless validation_set.unique_feature_type + "or all classification validations") unless validation_set.unique_feature_type raise OpenTox::BadRequestError.new("number of different identifiers <2: "+ validation_set.get_values(:identifier).inspect) if validation_set.num_different_values(:identifier)<2 -- cgit v1.2.3 From 9407fcd5c937370324764a08afd589f386e83b60 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 6 Jul 2011 14:57:53 +0200 Subject: fix error when making regression validations w/o confidence --- lib/predictions.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/predictions.rb b/lib/predictions.rb index 4e34c90..082ad10 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -146,9 +146,9 @@ module Lib delta = predicted_value - actual_value @sum_error += delta @sum_abs_error += delta.abs - @sum_weighted_abs_error += delta.abs*confidence_value + @sum_weighted_abs_error += delta.abs*confidence_value if @conf_provided @sum_squared_error += delta**2 - @sum_weighted_squared_error += (delta**2)*confidence_value + @sum_weighted_squared_error += (delta**2)*confidence_value if @conf_provided old_prediction_mean = @prediction_mean @prediction_mean = (@prediction_mean * (@num_predicted-1) + predicted_value) / @num_predicted.to_f -- cgit v1.2.3 From d0f50a2218fca7580be80e0595b0a73069329ad6 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 12 Jul 2011 11:41:32 +0200 Subject: fixed sqrt domain error in regression validation, added check for NaN and Infinity --- lib/predictions.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/predictions.rb b/lib/predictions.rb index 082ad10..f4fb461 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -59,7 +59,7 @@ module Lib raise "accept_values != nil while performing regression" if @accept_values { "predicted"=>@predicted_values, "actual"=>@actual_values }.each do |s,values| values.each{ |v| raise "illegal "+s+" regression-value ("+v.to_s+"),"+ - "has to be either nil or number" unless v==nil or v.is_a?(Numeric)} + "has to be either nil or number (not NaN, not Infinite)" if v!=nil and (!v.is_a?(Numeric) or v.nan? or v.finite?)} end end @@ -471,7 +471,9 @@ module Lib public def root_mean_squared_error return 0 if (@num_with_actual_value - @num_unpredicted)==0 - Math.sqrt(@sum_squared_error / (@num_with_actual_value - @num_unpredicted).to_f) + mse = @sum_squared_error / (@num_with_actual_value - @num_unpredicted).to_f + return 0 if mse.nan? + Math.sqrt(mse) end def weighted_root_mean_squared_error -- cgit v1.2.3 From 1a01946eb3c89b9a672fb2c1165236c84cbaef36 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 12 Jul 2011 13:21:27 +0200 Subject: regression plot in 2 different scales --- report/plot_factory.rb | 28 ++++++++++++++++++---------- report/report_content.rb | 38 +++++++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/report/plot_factory.rb b/report/plot_factory.rb index d6c7d35..26a9c67 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -52,11 +52,12 @@ module Reports module PlotFactory - def self.create_regression_plot( out_files, validation_set, name_attribute ) + def self.create_regression_plot( out_files, validation_set, name_attribute, logscale=true ) out_files = [out_files] unless out_files.is_a?(Array) LOGGER.debug "Creating regression plot, out-file:"+out_files.to_s + omit_count = 0 names = [] x = [] y = [] @@ -64,25 +65,32 @@ module Reports x_i = v.get_predictions.predicted_values y_i = v.get_predictions.actual_values - # filter out nil-predictions - not_nil_indices = [] + # filter out nil-predictions and <=0 predictions if log-scale wanted + valid_indices = [] x_i.size.times do |i| - not_nil_indices << i if x_i[i]!=nil && y_i[i]!=nil + if x_i[i]!=nil and y_i[i]!=nil and (!logscale or (x_i[i]>0 and y_i[i]>0)) + valid_indices << i + else + omit_count += 1 + end end - if not_nil_indices.size < x_i.size - x_i = not_nil_indices.collect{ |i| x_i[i] } - y_i = not_nil_indices.collect{ |i| y_i[i] } + if valid_indices.size < x_i.size + x_i = valid_indices.collect{ |i| x_i[i] } + y_i = valid_indices.collect{ |i| y_i[i] } end names << ( name_attribute==:crossvalidation_fold ? "fold " : "" ) + v.send(name_attribute).to_s x << x_i y << y_i end - - raise "no predictions performed" if x.size==0 || x[0].size==0 + names = [""] if names.size==1 + + omit_str = omit_count>0 ? " ("+omit_count.to_s+" predictions omitted)" : "" + raise "no predictions performed"+omit_str if x.size==0 || x[0].size==0 out_files.each do |out_file| - RubyPlot::regression_point_plot(out_file, "Regression plot", "Predicted values", "Actual values", names, x, y ) + RubyPlot::regression_point_plot(out_file, "Regression plot", "Predicted values", "Actual values", names, x, y, logscale) end + omit_count end diff --git a/report/report_content.rb b/report/report_content.rb index ea2ad5a..8c437a8 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -141,31 +141,39 @@ class Reports::ReportContent name_attribute, section_title="Regression Plot", section_text=nil, - image_title=nil) + image_title="Regression Plot") - image_title = "Regression plot" unless image_title #section_regr = @xml_report.add_section(@current_section, section_title) section_regr = @current_section prediction_set = validation_set.collect{ |v| v.get_predictions } if prediction_set.size>0 - section_text += "\nWARNING: regression plot information not available for all validation results" if prediction_set.size!=validation_set.size - @xml_report.add_paragraph(section_regr, section_text) if section_text - begin - plot_png = add_tmp_file("regr_plot", "png") - plot_svg = add_tmp_file("regr_plot", "svg") - Reports::PlotFactory.create_regression_plot( [plot_png[:path], plot_svg[:path]], prediction_set, name_attribute ) - @xml_report.add_imagefigure(section_regr, image_title, plot_png[:name], "PNG", 100, plot_svg[:name]) - rescue Exception => ex - LOGGER.error("Could not create regression plot: "+ex.message) - rm_tmp_file(plot_png[:name]) - rm_tmp_file(plot_svg[:name]) - @xml_report.add_paragraph(section_regr, "could not create regression plot: "+ex.message) - end + [true, false].each do |log| + scale_str = (log ? " (logarithmic scale)" : " (linear scale)") + image_title_2 = image_title + scale_str + section_title_2 = section_title + scale_str + + section_text += "\nWARNING: regression plot information not available for all validation results" if prediction_set.size!=validation_set.size + @xml_report.add_paragraph(section_regr, section_text) if section_text + begin + log_str = (log ? "_log" : "") + plot_png = add_tmp_file("regr_plot"+log_str, "png") + plot_svg = add_tmp_file("regr_plot"+log_str, "svg") + omit_count = Reports::PlotFactory.create_regression_plot( [plot_png[:path], plot_svg[:path]], prediction_set, name_attribute, log ) + image_title_2 += " ("+omit_count.to_s+" datapoints omitted)" if omit_count>0 + @xml_report.add_imagefigure(section_regr, image_title_2, plot_png[:name], "PNG", 100, plot_svg[:name]) + rescue Exception => ex + LOGGER.error("Could not create regression plot: "+ex.message) + rm_tmp_file(plot_png[:name]) + rm_tmp_file(plot_svg[:name]) + @xml_report.add_paragraph(section_regr, "could not create regression plot: "+ex.message) + end + end else @xml_report.add_paragraph(section_regr, "No prediction info for regression available.") end + align_last_two_images section_title+" in logarithmic and linear scale (values <= 0 are omitted in logarithmic scale)" end def add_roc_plot( validation_set, -- cgit v1.2.3 From 2909344873532a1c7cc2a5ac80a8222e9828bc99 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 12 Jul 2011 13:58:27 +0200 Subject: minor fix for new regression plotting: dont count nil values as omitted value --- report/plot_factory.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/report/plot_factory.rb b/report/plot_factory.rb index 26a9c67..bf59960 100644 --- a/report/plot_factory.rb +++ b/report/plot_factory.rb @@ -68,10 +68,12 @@ module Reports # filter out nil-predictions and <=0 predictions if log-scale wanted valid_indices = [] x_i.size.times do |i| - if x_i[i]!=nil and y_i[i]!=nil and (!logscale or (x_i[i]>0 and y_i[i]>0)) - valid_indices << i - else - omit_count += 1 + if x_i[i]!=nil and y_i[i]!=nil + if !logscale or (x_i[i]>0 and y_i[i]>0) + valid_indices << i + else + omit_count += 1 + end end end if valid_indices.size < x_i.size -- cgit v1.2.3 From 6600fd4d25c4bb34dfdf17b539d3598b112ffd2a Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 13 Jul 2011 10:15:47 +0200 Subject: fix valid prediction values check for regression validation --- lib/predictions.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/predictions.rb b/lib/predictions.rb index f4fb461..a4c57a0 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -55,11 +55,11 @@ module Lib values.each{ |v| raise "illegal "+s+" classification-value ("+v.to_s+"),"+ "has to be either nil or index of predicted-values" if v!=nil and (!v.is_a?(Numeric) or v<0 or v>@num_classes)} end - when "regresssion" + when "regression" raise "accept_values != nil while performing regression" if @accept_values { "predicted"=>@predicted_values, "actual"=>@actual_values }.each do |s,values| values.each{ |v| raise "illegal "+s+" regression-value ("+v.to_s+"),"+ - "has to be either nil or number (not NaN, not Infinite)" if v!=nil and (!v.is_a?(Numeric) or v.nan? or v.finite?)} + " has to be either nil or number (not NaN, not Infinite)" unless v==nil or (v.is_a?(Numeric) and !v.nan? and v.finite?)} end end -- cgit v1.2.3 From e0ff256a0a4508b2c4e250e7b02aa9b3099d9abb Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 13 Jul 2011 10:58:19 +0200 Subject: added correlation coefficient to regression reports --- report/report_factory.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/report/report_factory.rb b/report/report_factory.rb index 7ec2e1c..340f276 100755 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -8,7 +8,8 @@ VAL_ATTR_CV = [ :algorithm_uri, :dataset_uri, :num_folds, :crossvalidation_fold VAL_ATTR_CLASS = [ :num_instances, :num_unpredicted, :accuracy, :weighted_accuracy, :weighted_area_under_roc, :area_under_roc, :f_measure, :true_positive_rate, :true_negative_rate ] VAL_ATTR_REGR = [ :num_instances, :num_unpredicted, :root_mean_squared_error, - :weighted_root_mean_squared_error, :mean_absolute_error, :weighted_mean_absolute_error, :r_square, :weighted_r_square ] + :weighted_root_mean_squared_error, :mean_absolute_error, :weighted_mean_absolute_error, :r_square, :weighted_r_square, + :sample_correlation_coefficient ] #VAL_ATTR_BAR_PLOT_CLASS = [ :accuracy, :weighted_area_under_roc, # :area_under_roc, :f_measure, :true_positive_rate, :true_negative_rate ] -- cgit v1.2.3 From db4f1b61f588bfb59f161763922199c8a07d7a56 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Mon, 18 Jul 2011 09:34:52 +0200 Subject: resuce correlation coeffiction computation errors --- lib/predictions.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/predictions.rb b/lib/predictions.rb index a4c57a0..b71359d 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -517,11 +517,13 @@ module Lib end def sample_correlation_coefficient - # formula see http://en.wikipedia.org/wiki/Correlation_and_dependence#Pearson.27s_product-moment_coefficient - scc = ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) / - ( Math.sqrt( [0, @num_predicted * @sum_squares_actual - @sum_actual**2].max ) * - Math.sqrt( [0, @num_predicted * @sum_squares_predicted - @sum_predicted**2].max ) ) - ( scc.infinite? || scc.nan? ) ? 0 : scc + begin + # formula see http://en.wikipedia.org/wiki/Correlation_and_dependence#Pearson.27s_product-moment_coefficient + scc = ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) / + ( Math.sqrt( @num_predicted * @sum_squares_actual - @sum_actual**2 ) * + Math.sqrt( @num_predicted * @sum_squares_predicted - @sum_predicted**2 ) ) + ( scc.infinite? || scc.nan? ) ? 0 : scc + rescue; 0; end end def total_sum_of_squares -- cgit v1.2.3 From 9ca1bfaecbbe8b2941cdc6cd8a907448b17eeb85 Mon Sep 17 00:00:00 2001 From: mr Date: Mon, 1 Aug 2011 22:31:54 +0200 Subject: add subjectid to crossvalidation call --- validation/validation_application.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 1c466cc..d2dfef0 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -44,7 +44,8 @@ post '/crossvalidation/?' do task = OpenTox::Task.create( "Perform crossvalidation", url_for("/crossvalidation", :full) ) do |task| #, params cv_params = { :dataset_uri => params[:dataset_uri], :algorithm_uri => params[:algorithm_uri], - :loo => "false" } + :loo => "false", + :subjectid => params[:subjectid] } [ :num_folds, :random_seed ].each{ |sym| cv_params[sym] = params[sym] if params[sym] } cv_params[:stratified] = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] cv = Validation::Crossvalidation.create cv_params -- cgit v1.2.3