diff options
author | mguetlein <martin.guetlein@gmail.com> | 2011-05-24 16:14:37 +0200 |
---|---|---|
committer | mguetlein <martin.guetlein@gmail.com> | 2011-05-24 16:14:37 +0200 |
commit | 1c2ed2e04d5cbd1842e646e6429bcfdc0b1372dd (patch) | |
tree | 8a650344ad68343f43ad2968750ca4df7b670ace | |
parent | 2317a75d3e91a6a03992a8f19d6559323146a256 (diff) |
rewrite: adjust to new prediction dataset format from lazar, read confidence from ambit algorithms
-rw-r--r-- | lib/feature_util.rb | 66 | ||||
-rwxr-xr-x | lib/ot_predictions.rb | 83 | ||||
-rwxr-xr-x | lib/predictions.rb | 27 | ||||
-rwxr-xr-x | report/report_content.rb | 9 | ||||
-rwxr-xr-x | report/validation_access.rb | 20 | ||||
-rwxr-xr-x | report/validation_data.rb | 5 | ||||
-rwxr-xr-x | validation/validation_application.rb | 10 | ||||
-rwxr-xr-x | validation/validation_service.rb | 22 |
8 files changed, 163 insertions, 79 deletions
diff --git a/lib/feature_util.rb b/lib/feature_util.rb new file mode 100644 index 0000000..405d991 --- /dev/null +++ b/lib/feature_util.rb @@ -0,0 +1,66 @@ + +module Lib + class FeatureUtil + + # this derieves the predicted_variable and predicted_confidence in prediction dataset + # predicted_variable: the feature-uri of model predictions + # predicted_confidence: the feature-uri of the model prediction confidence + # according to API info should be available in the API + # problem: IST has no feature-service -> predicted_variable depends on prediction dataset + # + # PENDING: ambit and ist declare prediction features differently -> unify + # + def self.predicted_variables( model, prediction_dataset_uri, subjectid=nil ) + + predicted_variable = nil + predicted_confidence = nil + + if model.metadata[OT.predictedVariables] + predictedVariables = model.metadata[OT.predictedVariables] + if predictedVariables.is_a?(Array) + if (predictedVariables.size==1) + predicted_variable = predictedVariables[0] + elsif (predictedVariables.size==2) + # PENDING identify confidence + conf_index = -1 + predictedVariables.size.times do |i| + conf_index = i if OpenTox::Feature.find(predictedVariables[i]).metadata[DC.title]=~/(?i)confidence/ + end + raise "size=2, no confidence "+predictedVariables.inspect+" "+model.uri.to_s if conf_index==-1 + predicted_variable = predictedVariables[1-conf_index] + predicted_confidence = predictedVariables[conf_index] + else + raise "size>2 "+predictedVariables.inspect+" "+model.uri.to_s + end + else + raise "predictedVariables is no array" + end + end + + unless predicted_variable + d = OpenTox::Dataset.new prediction_dataset_uri + d.load_features(subjectid) + d.features.keys.each do |f| + if d.features[f][OT.hasSource]==model.uri + puts "source matching" + # PENDING identify confidence + if f =~ /(?i)confidence/ + puts "conf matiching" + raise "duplicate confidence feature, what to choose?" if predicted_confidence!=nil + predicted_confidence = f + elsif d.features[f][RDF.type].include? OT.ModelPrediction + puts "type include prediction" + raise "duplicate predicted variable, what to choose?" if predicted_variable!=nil + predicted_variable = f + end + end + puts d.features[f][OT.hasSource] + end + raise "could not estimate predicted variable" unless predicted_variable + end + + {:predicted_variable => predicted_variable, :predicted_confidence => predicted_confidence} + end + end +end + diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index 2e3c238..df1efc3 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -16,16 +16,18 @@ module Lib end def initialize( feature_type, test_dataset_uris, test_target_dataset_uris, - prediction_feature, prediction_dataset_uris, predicted_variables, subjectid=nil, task=nil) + prediction_feature, prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid=nil, task=nil) test_dataset_uris = [test_dataset_uris] unless test_dataset_uris.is_a?(Array) test_target_dataset_uris = [test_target_dataset_uris] unless test_target_dataset_uris.is_a?(Array) prediction_dataset_uris = [prediction_dataset_uris] unless prediction_dataset_uris.is_a?(Array) predicted_variables = [predicted_variables] unless predicted_variables.is_a?(Array) + predicted_confidences = [predicted_confidences] unless predicted_confidences.is_a?(Array) LOGGER.debug "loading prediciton -- test-dataset: "+test_dataset_uris.inspect LOGGER.debug "loading prediciton -- test-target-datset: "+test_target_dataset_uris.inspect LOGGER.debug "loading prediciton -- prediction-dataset: "+prediction_dataset_uris.inspect LOGGER.debug "loading prediciton -- predicted_variable: "+predicted_variables.inspect + LOGGER.debug "loading prediciton -- predicted_confidence: "+predicted_confidences.inspect LOGGER.debug "loading prediciton -- prediction_feature: "+prediction_feature.to_s raise "prediction_feature missing" unless prediction_feature @@ -46,6 +48,7 @@ module Lib test_target_dataset_uri = test_target_dataset_uris[i] prediction_dataset_uri = prediction_dataset_uris[i] predicted_variable = predicted_variables[i] + predicted_confidence = predicted_confidences[i] predicted_variable=prediction_feature if predicted_variable==nil @@ -72,7 +75,7 @@ module Lib "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil end - + compounds = test_dataset.compounds LOGGER.debug "test dataset size: "+compounds.size.to_s raise "test dataset is empty "+test_dataset_uri.to_s unless compounds.size>0 @@ -92,35 +95,24 @@ module Lib compounds.each do |c| case feature_type when "classification" - actual_values << classification_value(test_target_dataset, c, prediction_feature, accept_values) + actual_values << classification_val(test_target_dataset, c, prediction_feature, accept_values) when "regression" - actual_values << regression_value(test_target_dataset, c, prediction_feature) + actual_values << regression_val(test_target_dataset, c, prediction_feature) end end task.progress( task_status += task_step ) if task # loaded actual values prediction_dataset = Lib::DatasetCache.find prediction_dataset_uri,subjectid raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset - - # TODO: remove LAZAR_PREDICTION_DATASET_HACK - no_prediction_feature = prediction_dataset.features.keys.index(predicted_variable)==nil - if no_prediction_feature - one_entry_per_compound = true - compounds.each do |c| - if prediction_dataset.data_entries[c] and prediction_dataset.data_entries[c].size != 1 - one_entry_per_compound = false - break - end - end - msg = "prediction-feature not found: '"+predicted_variable+"' in prediction-dataset: "+prediction_dataset_uri.to_s+", available features: "+ - prediction_dataset.features.keys.inspect - if one_entry_per_compound - LOGGER.warn msg - else - raise msg - end - end - + raise "predicted_variable not found in prediction_dataset\n"+ + "predicted_variable '"+predicted_variable.to_s+"'\n"+ + "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+ + "available features are: "+prediction_dataset.features.inspect if prediction_dataset.features.keys.index(predicted_variable)==nil + raise "predicted_confidence not found in prediction_dataset\n"+ + "predicted_confidence '"+predicted_confidence.to_s+"'\n"+ + "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+ + "available features are: "+prediction_dataset.features.inspect if predicted_confidence and prediction_dataset.features.keys.index(predicted_confidence)==nil + raise "more predicted than test compounds, #test: "+compounds.size.to_s+" < #prediction: "+ prediction_dataset.compounds.size.to_s+", test-dataset: "+test_dataset_uri.to_s+", prediction-dataset: "+ prediction_dataset_uri if compounds.size < prediction_dataset.compounds.size @@ -141,25 +133,18 @@ module Lib else case feature_type when "classification" - # TODO: remove LAZAR_PREDICTION_DATASET_HACK - predicted_values << classification_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable, accept_values) + predicted_values << classification_val(prediction_dataset, c, predicted_variable, accept_values) when "regression" - predicted_values << regression_value(prediction_dataset, c, no_prediction_feature ? nil : predicted_variable) + predicted_values << regression_val(prediction_dataset, c, predicted_variable) end - # TODO confidence_values << prediction_dataset.get_prediction_confidence(c, predicted_variable) - conf = predicted_values[count]!=nil ? 1 : nil - begin - feature = prediction_dataset.data_entries[c].keys[0] - feature_data = prediction_dataset.features[feature] - conf = feature_data[OT.confidence].to_f if feature_data[OT.confidence]!=nil - rescue - LOGGER.warn "could not get confidence" + if predicted_confidence + confidence_values << confidence_val(prediction_dataset, c, predicted_confidence) + else + confidence_values << nil end - confidence_values << conf end count += 1 end - @compounds += compounds all_predicted_values += predicted_values all_actual_values += actual_values @@ -174,7 +159,7 @@ module Lib end private - def regression_value(dataset, compound, feature) + def regression_val(dataset, compound, feature) v = value(dataset, compound, feature) begin v = v.to_f unless v==nil or v.is_a?(Numeric) @@ -185,7 +170,18 @@ module Lib end end - def classification_value(dataset, compound, feature, accept_values) + def confidence_val(dataset, compound, confidence) + v = value(dataset, compound, confidence) + begin + v = v.to_f unless v==nil or v.is_a?(Numeric) + v + rescue + LOGGER.warn "no numeric value for confidence '"+v.to_s+"'" + nil + end + end + + def classification_val(dataset, compound, feature, accept_values) v = value(dataset, compound, feature) i = accept_values.index(v.to_s) raise "illegal class_value of prediction (value is '"+v.to_s+"'), accept values are "+ @@ -239,6 +235,10 @@ module Lib def self.to_array( predictions, add_pic=false, format=false ) + confidence_available = false + predictions.each do |p| + confidence_available |= p.confidence_values_available? + end res = [] conf_column = nil predictions.each do |p| @@ -269,9 +269,9 @@ module Lib a << nil end end - if p.confidence_values_available? + if confidence_available conf_column = a.size if conf_column==nil - a << p.confidence_value(i) #(format ? p.confidence_value(i).to_nice_s : p.confidence_value(i)) + a << p.confidence_value(i) end a << p.identifier(i) res << a @@ -279,6 +279,7 @@ module Lib end if conf_column!=nil + LOGGER.debug "sort via confidence: "+res.collect{|n| n[conf_column]}.inspect res = res.sort_by{ |n| n[conf_column] || 0 }.reverse if format res.each do |a| diff --git a/lib/predictions.rb b/lib/predictions.rb index 4a15e1d..a449776 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -45,16 +45,6 @@ module Lib raise "illegal num confidence values "+num_info if @confidence_values.size != @predicted_values.size @confidence_values.each{ |c| raise "illegal confidence value: '"+c.to_s+"'" unless c==nil or (c.is_a?(Numeric) and c>=0 and c<=1) } - ## check if there is more than one different conf value - ## DEPRECATED? not sure anymore what this was about, - ## I am pretty sure this was for r-plot of roc curves - ## roc curvers are now plotted manually - #conf_val_tmp = {} - #@confidence_values.each{ |c| conf_val_tmp[c] = nil } - #if conf_val_tmp.keys.size<2 - # LOGGER.warn("prediction w/o confidence values"); - # @confidence_values=nil - #end case @feature_type when "classification" @@ -75,11 +65,13 @@ module Lib init_stats() (0..@predicted_values.size-1).each do |i| - update_stats( @predicted_values[i], @actual_values[i], (@confidence_values!=nil)?@confidence_values[i]:nil ) + update_stats( @predicted_values[i], @actual_values[i], @confidence_values[i] ) end end def init_stats + @conf_provided = false + @num_no_actual_value = 0 @num_with_actual_value = 0 @@ -134,6 +126,8 @@ module Lib else @num_predicted += 1 + @conf_provided |= confidence_value!=nil + case @feature_type when "classification" @confusion_matrix[actual_value][predicted_value] += 1 @@ -186,6 +180,7 @@ module Lib end def weighted_accuracy + return 0 unless confidence_values_available? raise "no classification" unless @feature_type=="classification" total = 0 correct = 0 @@ -255,7 +250,7 @@ module Lib def area_under_roc(class_index=nil) return prediction_feature_value_map( lambda{ |i| area_under_roc(i) } ) if class_index==nil - return 0.0 if @confidence_values==nil + return 0 unless confidence_values_available? LOGGER.warn("TODO: implement approx computiation of AUC,"+ "so far Wilcoxon-Man-Whitney is used (exponential)") if @@ -485,7 +480,9 @@ module Lib # see http://en.wikipedia.org/wiki/Coefficient_of_determination#Definitions # see http://web.maths.unsw.edu.au/~adelle/Garvan/Assays/GoodnessOfFit.html - r_2 = 1 - residual_sum_of_squares / total_sum_of_squares + ss_tot = total_sum_of_squares + return 0 if ss_tot==0 + r_2 = 1 - residual_sum_of_squares / ss_tot ( r_2.infinite? || r_2.nan? ) ? 0 : r_2 end @@ -523,7 +520,7 @@ module Lib def get_prediction_values(class_value) #puts "get_roc_values for class_value: "+class_value.to_s - raise "no confidence values" if @confidence_values==nil + raise "no confidence values" unless confidence_values_available? #raise "no class-value specified" if class_value==nil class_index = @accept_values.index(class_value) if class_value!=nil @@ -594,7 +591,7 @@ module Lib end def confidence_values_available? - return @confidence_values!=nil + @conf_provided end ################################################################################################################### diff --git a/report/report_content.rb b/report/report_content.rb index d60f700..755147d 100755 --- a/report/report_content.rb +++ b/report/report_content.rb @@ -287,12 +287,9 @@ class Reports::ReportContent section_bar = @xml_report.add_section(@current_section, section_title) @xml_report.add_paragraph(section_bar, section_text) if section_text - - plot_file_name = "bar_plot"+@tmp_file_count.to_s+".png" - @tmp_file_count += 1 - plot_file_path = add_tmp_file(plot_file_name) - Reports::PlotFactory.create_bar_plot(plot_file_path, validation_set, title_attribute, value_attributes ) - @xml_report.add_imagefigure(section_bar, image_title, plot_file_name, "PNG", 100, image_caption) + plot_bar = add_tmp_file("bar_plot", "svg") + Reports::PlotFactory.create_bar_plot(plot_bar[:path], validation_set, title_attribute, value_attributes ) + @xml_report.add_imagefigure(section_bar, image_title, plot_bar[:name], "PNG", 100, image_caption) end private diff --git a/report/validation_access.rb b/report/validation_access.rb index ffb7461..d421a0b 100755 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -100,15 +100,17 @@ class Reports::ValidationDB def get_predictions(validation, subjectid=nil, task=nil) Lib::OTPredictions.new( validation.feature_type, validation.test_dataset_uri, validation.test_target_dataset_uri, validation.prediction_feature, validation.prediction_dataset_uri, - validation.predicted_variable, subjectid, task) + validation.predicted_variable, validation.predicted_confidence, subjectid, task) end def get_accept_values( validation, subjectid=nil ) # PENDING So far, one has to load the whole dataset to get the accept_value from ambit - d = Lib::DatasetCache.find( validation.test_target_dataset_uri, subjectid ) - raise "cannot get test target dataset for accept values, dataset: "+validation.test_target_dataset_uri.to_s unless d + test_target_dataset = validation.test_target_dataset_uri + test_target_dataset = validation.test_dataset_uri unless test_target_dataset + d = Lib::DatasetCache.find( test_target_dataset, subjectid ) + raise "cannot get test target dataset for accept values, dataset: "+test_target_dataset.to_s unless d accept_values = d.features[validation.prediction_feature][OT.acceptValue] - raise "cannot get accept values from dataset "+validation.test_target_dataset_uri.to_s+" for feature "+ + raise "cannot get accept values from dataset "+test_target_dataset.to_s+" for feature "+ validation.prediction_feature+":\n"+d.features[validation.prediction_feature].to_yaml unless accept_values!=nil accept_values end @@ -122,10 +124,16 @@ class Reports::ValidationDB raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) model = OpenTox::Model::Generic.find(validation.model_uri, subjectid) raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model - model.metadata[OT.predictedVariables] - #get_model(validation).predictedVariables + Lib::FeatureUtil.predicted_variables(model, validation.prediction_dataset_uri, subjectid)[:predicted_variable] end + def predicted_confidence(validation, subjectid=nil) + raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) + model = OpenTox::Model::Generic.find(validation.model_uri, subjectid) + raise OpenTox::NotFoundError.new "model not found '"+validation.model_uri+"'" unless model + Lib::FeatureUtil.predicted_variables(model, validation.prediction_dataset_uri, subjectid)[:predicted_confidence] + end + # private # def get_model(validation) # raise "cannot derive model depended props for merged validations" if Lib::MergeObjects.merged?(validation) diff --git a/report/validation_data.rb b/report/validation_data.rb index 0c15dd2..2479977 100755 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -139,6 +139,11 @@ module Reports @predicted_variable = Reports.validation_access.predicted_variable(self, @subjectid) end + def predicted_confidence + return @predicted_confidence if @predicted_confidence!=nil + @predicted_confidence = Reports.validation_access.predicted_confidence(self, @subjectid) + end + # loads all crossvalidation attributes, of the corresponding cv into this object def load_cv_attributes raise "crossvalidation-id not set" unless @crossvalidation_id diff --git a/validation/validation_application.rb b/validation/validation_application.rb index d448c62..f64f74e 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -4,6 +4,7 @@ end require 'lib/dataset_cache.rb' +require 'lib/feature_util.rb' require 'validation/validation_service.rb' get '/crossvalidation/?' do @@ -443,21 +444,22 @@ post '/validate_datasets' do params[:validation_type] = "validate_datasets" if params[:model_uri] + raise OpenTox::BadRequestError.new "please specify 'model_uri' or set either 'classification' or 'regression' flag" if params[:classification] or params[:regression] v = Validation::Validation.create params v.subjectid = @subjectid v.compute_validation_stats_with_model(nil,false,task) else raise OpenTox::BadRequestError.new "please specify 'model_uri' or 'prediction_feature'" unless params[:prediction_feature] - raise OpenTox::BadRequestError.new "please specify 'model_uri' or 'predicted_feature'" unless params[:predicted_feature] + raise OpenTox::BadRequestError.new "please specify 'model_uri' or 'predicted_variable'" unless params[:predicted_variable] raise OpenTox::BadRequestError.new "please specify 'model_uri' or set either 'classification' or 'regression' flag" unless params[:classification] or params[:regression] - - predicted_feature = params.delete("predicted_feature") + predicted_variable = params.delete("predicted_variable") + predicted_confidence = params.delete("predicted_confidence") feature_type = "classification" if params.delete("classification")!=nil feature_type = "regression" if params.delete("regression")!=nil v = Validation::Validation.create params v.subjectid = @subjectid - v.compute_validation_stats(feature_type,predicted_feature,nil,nil,false,task) + v.compute_validation_stats(feature_type,predicted_variable,predicted_confidence,nil,nil,false,task) end v.validation_uri end diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 61f3a6e..73c15df 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -46,9 +46,15 @@ module Validation test_target_dataset_uris = vals.collect{|v| v.test_target_dataset_uri} prediction_feature = vals.first.prediction_feature prediction_dataset_uris = vals.collect{|v| v.prediction_dataset_uri} - predicted_variables = models.collect{|m| m.metadata[OT.predictedVariables]} + predicted_variables = [] + predicted_confidences = [] + models.size.times do |i| + predicted = Lib::FeatureUtil.predicted_variables(models[i], prediction_dataset_uris[i], subjectid) + predicted_variables << predicted[:predicted_variable] + predicted_confidences << predicted[:predicted_confidence] + end prediction = Lib::OTPredictions.new( feature_type, test_dataset_uris, test_target_dataset_uris, prediction_feature, - prediction_dataset_uris, predicted_variables, subjectid ) + prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid ) v = Validation.new case feature_type @@ -218,13 +224,15 @@ module Validation dependentVariables = model.metadata[OT.dependentVariables] prediction_feature = self.prediction_feature ? nil : dependentVariables algorithm_uri = self.algorithm_uri ? nil : model.metadata[OT.algorithm] - predictedVariables = model.metadata[OT.predictedVariables] - compute_validation_stats( model.feature_type(self.subjectid), predictedVariables, + predicted_variables = Lib::FeatureUtil.predicted_variables(model, prediction_dataset_uri, subjectid) + predicted_variable = predicted_variables[:predicted_variable] + predicted_confidence = predicted_variables[:predicted_confidence] + compute_validation_stats( model.feature_type(self.subjectid), predicted_variable, predicted_confidence, prediction_feature, algorithm_uri, dry_run, task ) end - def compute_validation_stats( feature_type, predicted_feature, prediction_feature=nil, - algorithm_uri=nil, dry_run=false, task=nil ) + def compute_validation_stats( feature_type, predicted_variable, predicted_confidence, prediction_feature, + algorithm_uri, dry_run, task ) # self.attributes = { :prediction_feature => prediction_feature } if self.prediction_feature==nil && prediction_feature # self.attributes = { :algorithm_uri => algorithm_uri } if self.algorithm_uri==nil && algorithm_uri @@ -237,7 +245,7 @@ module Validation LOGGER.debug "computing prediction stats" prediction = Lib::OTPredictions.new( feature_type, self.test_dataset_uri, self.test_target_dataset_uri, self.prediction_feature, - self.prediction_dataset_uri, predicted_feature, self.subjectid, OpenTox::SubTask.create(task, 0, 80) ) + self.prediction_dataset_uri, predicted_variable, predicted_confidence, self.subjectid, OpenTox::SubTask.create(task, 0, 80) ) #reading datasets and computing the main stats is 80% the work unless dry_run |