From 26c0b93a02fddb60175747f7733d13e973257cd8 Mon Sep 17 00:00:00 2001 From: mr Date: Tue, 1 Feb 2011 16:34:20 +0100 Subject: A&A for validations --- lib/validation.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib/validation.rb') diff --git a/lib/validation.rb b/lib/validation.rb index 76c4529..23b246b 100644 --- a/lib/validation.rb +++ b/lib/validation.rb @@ -13,18 +13,18 @@ module OpenTox OpenTox::Validation.new(uri) end - def create_report - @report_uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/report/crossvalidation"), :validation_uris => @uri).to_s + def create_report(subjectid=nil) + @report_uri = OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/report/crossvalidation"), {:validation_uris => @uri, :subjectid => subjectid}).to_s @report_uri end - def create_qmrf_report - @qmrf_report_uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/reach_report/qmrf"), :model_uri => @uri).to_s + def create_qmrf_report(subjectid=nil) + @qmrf_report_uri = OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/reach_report/qmrf"), {:model_uri => @uri, :subjectid => subjectid}).to_s @qmrf_report_uri end - def summary(type) - v = YAML.load RestClientWrappper.get(File.join(@uri, 'statistics'),:accept => "application/x-yaml").to_s + def summary(type, subjectid=nil) + v = YAML.load OpenTox::RestClientWrapper.get(File.join(@uri, 'statistics'),{:accept => "application/x-yaml", :subjectid => subjectid}).to_s case type when "classification" -- cgit v1.2.3 From e4a32e37ab8708aa8ae4dbfc6069e5ea75928f3f Mon Sep 17 00:00:00 2001 From: mr Date: Tue, 8 Feb 2011 12:58:30 +0100 Subject: manually insert code from mguetlein repository | restclientwrapper.post event with changed method call --- lib/validation.rb | 240 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 183 insertions(+), 57 deletions(-) (limited to 'lib/validation.rb') diff --git a/lib/validation.rb b/lib/validation.rb index 23b246b..2ea19a9 100644 --- a/lib/validation.rb +++ b/lib/validation.rb @@ -1,70 +1,196 @@ module OpenTox - class Validation + class Validation include OpenTox - - attr_accessor :report_uri, :qmrf_report_uri - - def self.create_crossvalidation(params) - params[:uri] = File.join(CONFIG[:services]['opentox-validation'], "crossvalidation") - params[:num_folds] = 10 unless params[:num_folds] - params[:random_seed] = 2 unless params[:random_seed] - params[:stratified] = false unless params[:stratified] - uri = OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/crossvalidation"),params,nil,false) - OpenTox::Validation.new(uri) - end - - def create_report(subjectid=nil) - @report_uri = OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/report/crossvalidation"), {:validation_uris => @uri, :subjectid => subjectid}).to_s - @report_uri + + # find validation, raises error if not found + # @param [String] uri + # @param [String,optional] subjectid + # @return [OpenTox::Validation] + def self.find( uri, subjectid=nil ) + val = Validation.new(uri) + val.load_metadata( subjectid ) + val end - - def create_qmrf_report(subjectid=nil) - @qmrf_report_uri = OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/reach_report/qmrf"), {:model_uri => @uri, :subjectid => subjectid}).to_s - @qmrf_report_uri + + # creates a validation object from crossvaldiation statistics, raise error if not found + # (as crossvaldiation statistics are returned as an average valdidation over all folds) + # @param [String] crossvalidation uri + # @param [String,optional] subjectid + # @return [OpenTox::Validation] + def self.from_cv_statistics( crossvalidation_uri, subjectid=nil ) + find( File.join(crossvalidation_uri, 'statistics'),subjectid ) end - - def summary(type, subjectid=nil) - v = YAML.load OpenTox::RestClientWrapper.get(File.join(@uri, 'statistics'),{:accept => "application/x-yaml", :subjectid => subjectid}).to_s - - case type - when "classification" - tp=0; tn=0; fp=0; fn=0; n=0 - v[:classification_statistics][:confusion_matrix][:confusion_matrix_cell].each do |cell| - if cell[:confusion_matrix_predicted] == "true" and cell[:confusion_matrix_actual] == "true" - tp = cell[:confusion_matrix_value] - n += tp - elsif cell[:confusion_matrix_predicted] == "false" and cell[:confusion_matrix_actual] == "false" - tn = cell[:confusion_matrix_value] - n += tn - elsif cell[:confusion_matrix_predicted] == "false" and cell[:confusion_matrix_actual] == "true" - fn = cell[:confusion_matrix_value] - n += fn - elsif cell[:confusion_matrix_predicted] == "true" and cell[:confusion_matrix_actual] == "false" - fp = cell[:confusion_matrix_value] - n += fp + + # loads metadata via yaml from validation object + # fields (like for example the validated model) can be acces via validation.metadata[OT.model] + def load_metadata( subjectid=nil ) + @metadata = YAML.load(OpenTox::RestClientWrapper.get(uri,{:subjectid => subjectid, :accept => "application/x-yaml"})) + end + + # PENDING: creates summary as used for ToxCreate + def summary + if @metadata[OT.classificationStatistics] + res = { + :nr_predictions => @metadata[OT.numInstances] - @metadata[OT.numUnpredicted], + :correct_predictions => @metadata[OT.classificationStatistics][OT.percentCorrect], + :weighted_area_under_roc => @metadata[OT.classificationStatistics][OT.weightedAreaUnderRoc], + } + @metadata[OT.classificationStatistics][OT.classValueStatistics].each do |s| + if s[OT.classValue].to_s=="true" + res[:true_positives] = s[OT.numTruePositives] + res[:false_positives] = s[OT.numFalsePositives] + res[:true_negatives] = s[OT.numTrueNegatives] + res[:false_negatives] = s[OT.numFalseNegatives] + res[:sensitivity] = s[OT.truePositiveRate] + res[:specificity] = s[OT.falsePositiveRate] + break end end + res + elsif @metadata[OT.regressionStatistics] { - :nr_predictions => n, - :true_positives => tp, - :false_positives => fp, - :true_negatives => tn, - :false_negatives => fn, - :correct_predictions => 100*(tp+tn).to_f/n, - :weighted_area_under_roc => v[:classification_statistics][:weighted_area_under_roc].to_f, - :sensitivity => tp.to_f/(tp+fn), - :specificity => tn.to_f/(tn+fp), - } - when "regression" - { - :nr_predictions => v[:num_instances] - v[:num_unpredicted], - :r_square => v[:regression_statistics][:r_square], - :root_mean_squared_error => v[:regression_statistics][:root_mean_squared_error], - :mean_absolute_error => v[:regression_statistics][:mean_absolute_error], + :nr_predictions => @metadata[OT.numInstances] - @metadata[OT.numUnpredicted], + :r_square => @metadata[OT.regressionStatistics][OT.rSquare], + :root_mean_squared_error => @metadata[OT.regressionStatistics][OT.rootMeanSquaredError], + :mean_absolute_error => @metadata[OT.regressionStatistics][OT.meanAbsoluteError], } end end + end + + class Crossvalidation + include OpenTox + + attr_reader :report + + # find crossvalidation, raises error if not found + # @param [String] uri + # @param [String,optional] subjectid + # @return [OpenTox::Crossvalidation] + def self.find( uri, subjectid=nil ) + cv = Crossvalidation.new(uri) + cv.load_metadata( subjectid ) + cv + end + + # creates a crossvalidations, waits until it finishes, may take some time + # @param [Hash] params (required:algorithm_uri,dataset_uri,prediction_feature, optional:algorithm_params,num_folds(10),random_seed(1),stratified(false)) + # @param [String,optional] subjectid + # @param [OpenTox::Task,optional] waiting_task (can be a OpenTox::Subtask as well), progress is updated accordingly + # @return [OpenTox::Crossvalidation] + def self.create( params, subjectid=nil, waiting_task=nil ) + params[:subjectid] = subjectid if subjectid + uri = OpenTox::RestClientWrapper.post( File.join(CONFIG[:services]["opentox-validation"],"crossvalidation"), + params,{:content_type => "text/uri-list"},waiting_task ) + Crossvalidation.new(uri) + end - end + # looks for report for this crossvalidation, creates a report if no report is found + # @param [String,optional] subjectid + # @param [OpenTox::Task,optional] waiting_task (can be a OpenTox::Subtask as well), progress is updated accordingly + # @return [OpenTox::CrossvalidationReport] + def find_or_create_report( subjectid=nil, waiting_task=nil ) + @report = CrossvalidationReport.find_for_crossvalidation(@uri, subjectid) unless @report + @report = CrossvalidationReport.create(@uri, subjectid, waiting_task) unless @report + @report.uri + end + + # loads metadata via yaml from crossvalidation object + # fields (like for example the validations) can be acces via validation.metadata[OT.validation] + def load_metadata( subjectid=nil ) + @metadata = YAML.load(OpenTox::RestClientWrapper.get(uri,{:subjectid => subjectid, :accept => "application/x-yaml"})) + end + + # PENDING: creates summary as used for ToxCreate + def summary( subjectid=nil ) + Validation.from_cv_statistics( @uri, subjectid ).summary + end + end + + class ValidationReport + include OpenTox + + # finds ValidationReport for a particular validation + # @param [String] crossvalidation uri + # @param [String,optional] subjectid + # @return [OpenTox::ValidationReport] nil if no report found + def self.find_for_validation( validation_uri, subjectid=nil ) + uris = RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"], + "/report/validation?validation="+validation_uri), {:subjectid => subjectid}).chomp.split("\n") + uris.size==0 ? nil : ValidationReport.new(uris[-1]) + end + + end + + class CrossvalidationReport + include OpenTox + + # finds CrossvalidationReport via uri, raises error if not found + # @param [String] uri + # @param [String,optional] subjectid + # @return [OpenTox::CrossvalidationReport] + def self.find( uri, subjectid=nil ) + # PENDING load report data? + OpenTox::RestClientWrapper.get(uri,{:subjectid => subjectid}) + CrossvalidationReport.new(uri) + end + + # finds CrossvalidationReport for a particular crossvalidation + # @param [String] crossvalidation uri + # @param [String,optional] subjectid + # @return [OpenTox::CrossvalidationReport] nil if no report found + def self.find_for_crossvalidation( crossvalidation_uri, subjectid=nil ) + uris = RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"], + "/report/crossvalidation?crossvalidation="+crossvalidation_uri), {:subjectid => subjectid}).chomp.split("\n") + uris.size==0 ? nil : CrossvalidationReport.new(uris[-1]) + end + + # creates a crossvalidation report via crossvalidation + # @param [String] crossvalidation uri + # @param [String,optional] subjectid + # @param [OpenTox::Task,optional] waiting_task (can be a OpenTox::Subtask as well), progress is updated accordingly + # @return [OpenTox::CrossvalidationReport] + def self.create( crossvalidation_uri, subjectid=nil, waiting_task=nil ) + uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/report/crossvalidation"), + { :validation_uris => crossvalidation_uri, :subjectid => subjectid }, {}, waiting_task ) + CrossvalidationReport.new(uri) + end + end + + class QMRFReport + include OpenTox + + # finds QMRFReport, raises Error if not found + # @param [String] uri + # @param [String,optional] subjectid + # @return [OpenTox::QMRFReport] + def self.find( uri, subjectid=nil ) + # PENDING load crossvalidation data? + OpenTox::RestClientWrapper.get(uri,{:subjectid => subjectid}) + QMRFReport.new(uri) + end + + # finds QMRF report for a particular model + # @param [String] model_uri + # @param [String,optional] subjectid + # @return [OpenTox::QMRFReport] nil if no report found + def self.find_for_model( model_uri, subjectid=nil ) + uris = RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"], + "/reach_report/qmrf?model="+model_uri), {:subjectid => subjectid}).chomp.split("\n") + uris.size==0 ? nil : QMRFReport.new(uris[-1]) + end + + # creates a qmrf report via model + # @param [String] model_uri + # @param [String,optional] subjectid + # @param [OpenTox::Task,optional] waiting_task (can be a OpenTox::Subtask as well), progress is updated accordingly + # @return [OpenTox::QMRFReport] + def self.create( model_uri, subjectid=nil, waiting_task=nil ) + uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/reach_report/qmrf"), + { :model_uri => model_uri, :subjectid => subjectid }, {}, waiting_task ) + QMRFReport.new(uri) + end + end + end -- cgit v1.2.3