From 34ef45bf1f87c787e3ddaccc03a36a5fa2d54c7f Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 3 Feb 2011 18:10:48 +0100 Subject: update /refactor validation.rb --- lib/algorithm.rb | 2 +- lib/rest_client_wrapper.rb | 7 +- lib/task.rb | 3 +- lib/validation.rb | 180 ++++++++++++++++++++++++++++++++------------- 4 files changed, 134 insertions(+), 58 deletions(-) diff --git a/lib/algorithm.rb b/lib/algorithm.rb index ae05e16..bfa9860 100644 --- a/lib/algorithm.rb +++ b/lib/algorithm.rb @@ -34,7 +34,7 @@ module OpenTox # Find Generic Opentox Algorithm via URI, and loads metadata # @param [String] uri Algorithm URI # @return [OpenTox::Algorithm::Generic] Algorithm instance, nil if alogrithm was not found - def self.find(uri, subjectid) + def self.find(uri, subjectid=nil) return nil unless uri alg = Generic.new(uri) alg.load_metadata( subjectid ) diff --git a/lib/rest_client_wrapper.rb b/lib/rest_client_wrapper.rb index f59dce7..658f111 100644 --- a/lib/rest_client_wrapper.rb +++ b/lib/rest_client_wrapper.rb @@ -55,11 +55,12 @@ module OpenTox raise OpenTox::BadRequestError.new "uri is null" unless uri raise OpenTox::BadRequestError.new "not a uri: "+uri.to_s unless uri.to_s.uri? - raise OpenTox::BadRequestError.new "headers are no hash: "+headers.inspect unless headers==nil or headers.is_a?(Hash) - raise OpenTox::BadRequestError.new "nil headers for post not allowed, use {}" if rest_call=="post" and headers==nil - headers.each{ |k,v| headers.delete(k) if v==nil } if headers #remove keys with empty values, as this can cause problems + raise "headers are no hash: "+headers.inspect unless headers==nil or headers.is_a?(Hash) raise OpenTox::BadRequestError.new "accept should go into the headers" if payload and payload.is_a?(Hash) and payload[:accept] raise OpenTox::BadRequestError.new "content_type should go into the headers" if payload and payload.is_a?(Hash) and payload[:content_type] + raise "__waiting_task__ must be 'nil' or '(sub)task', is "+waiting_task.class.to_s if + waiting_task!=nil and !(waiting_task.is_a?(Task) || waiting_task.is_a?(SubTask)) + headers.each{ |k,v| headers.delete(k) if v==nil } if headers #remove keys with empty values, as this can cause problems # PENDING needed for NUTA, until we finally agree on how to send subjectid headers[:subjectid] = payload.delete(:subjectid) if uri=~/ntua/ and payload and payload.is_a?(Hash) and payload.has_key?(:subjectid) diff --git a/lib/task.rb b/lib/task.rb index f635b43..9c52299 100644 --- a/lib/task.rb +++ b/lib/task.rb @@ -33,7 +33,7 @@ module OpenTox def self.create( title=nil, creator=nil, max_duration=DEFAULT_TASK_MAX_DURATION, description=nil ) params = {:title=>title, :creator=>creator, :max_duration=>max_duration, :description=>description } - task_uri = RestClientWrapper.post(CONFIG[:services]["opentox-task"], params, {}, false).to_s + task_uri = RestClientWrapper.post(CONFIG[:services]["opentox-task"], params, {}, nil, false).to_s task = Task.new(task_uri.chomp) # measure current memory consumption @@ -284,7 +284,6 @@ module OpenTox raise OpenTox::BadRequestError.new ex.message+" (task-uri:"+@uri+")" end end - end # Convenience class to split a (sub)task into subtasks diff --git a/lib/validation.rb b/lib/validation.rb index 83be91a..b1ccb7b 100644 --- a/lib/validation.rb +++ b/lib/validation.rb @@ -1,70 +1,146 @@ module OpenTox - class Validation + class Crossvalidation include OpenTox - attr_accessor :report_uri, :qmrf_report_uri + attr_reader :report + + # find crossvalidation, raises error if not found + # @param [String] uri + # @param [String,optional] subjectid + # @return [OpenTox::Crossvalidation] + def self.find( uri, subjectid=nil ) + # PENDING load crossvalidation data? + OpenTox::RestClientWrapper.get(uri,{:subjectid => subjectid}) + Crossvalidation.new(uri) + end - def self.create_crossvalidation(params) - params[:uri] = File.join(CONFIG[:services]['opentox-validation'], "crossvalidation") + # creates a crossvalidations, waits until it finishes, may take some time + # @param [Hash] params + # @param [String,optional] subjectid + # @param [OpenTox::Task,optional] waiting_task (can be a OpenTox::Subtask as well), progress is updated accordingly + # @return [OpenTox::Crossvalidation] + def self.create( params, subjectid=nil, waiting_task=nil ) + params[:uri] = File.join(CONFIG[:services]['opentox-validation'], "crossvalidation") params[:num_folds] = 10 unless params[:num_folds] - params[:random_seed] = 2 unless params[:random_seed] - params[:stratified] = false unless params[:stratified] - uri = OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/crossvalidation"),params,{},false) - OpenTox::Validation.new(uri) - end - - def create_report - @report_uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/report/crossvalidation"), :validation_uris => @uri).to_s - @report_uri + params[:random_seed] = 2 unless params[:random_seed] + params[:stratified] = false unless params[:stratified] + params[:subjectid] = subjectid if subjectid + uri = OpenTox::RestClientWrapper.post( File.join(CONFIG[:services]["opentox-validation"],"/crossvalidation"), + params,{:content_type => "text/uri-list"},waiting_task ) + Crossvalidation.new(uri) end - def create_qmrf_report - @qmrf_report_uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/reach_report/qmrf"), :model_uri => @uri).to_s - @qmrf_report_uri + # looks for report for this crossvalidation, creates a report if no report is found + # @param [String,optional] subjectid + # @param [OpenTox::Task,optional] waiting_task (can be a OpenTox::Subtask as well), progress is updated accordingly + # @return [OpenTox::CrossvalidationReport] + def find_or_create_report( subjectid=nil, waiting_task=nil ) + @report = CrossvalidationReport.find_for_crossvalidation(self, subjectid) unless @report + @report = CrossvalidationReport.create(self, subjectid, waiting_task) unless @report + @report end - def summary(type) - v = YAML.load RestClientWrappper.get(File.join(@uri, 'statistics'),:accept => "application/x-yaml").to_s - - case type - when "classification" - tp=0; tn=0; fp=0; fn=0; n=0 - v[:classification_statistics][:confusion_matrix][:confusion_matrix_cell].each do |cell| - if cell[:confusion_matrix_predicted] == "true" and cell[:confusion_matrix_actual] == "true" - tp = cell[:confusion_matrix_value] - n += tp - elsif cell[:confusion_matrix_predicted] == "false" and cell[:confusion_matrix_actual] == "false" - tn = cell[:confusion_matrix_value] - n += tn - elsif cell[:confusion_matrix_predicted] == "false" and cell[:confusion_matrix_actual] == "true" - fn = cell[:confusion_matrix_value] - n += fn - elsif cell[:confusion_matrix_predicted] == "true" and cell[:confusion_matrix_actual] == "false" - fp = cell[:confusion_matrix_value] - n += fp + # PENDING: creates summary as used for ToxCreate + def summary + v = YAML.load RestClientWrapper.get(File.join(@uri, 'statistics'),:accept => "application/x-yaml").to_s + if v[OT.classificationStatistics] + res = { + :nr_predictions => v[OT.numInstances] - v[OT.numUnpredicted], + :correct_predictions => v[OT.classificationStatistics][OT.percentCorrect], + :weighted_area_under_roc => v[OT.classificationStatistics][OT.weightedAreaUnderRoc], + } + v[OT.classificationStatistics][OT.classValueStatistics].each do |s| + if s[OT.classValue].to_s=="true" + res[:true_positives] = s[OT.numTruePositives] + res[:false_positives] = s[OT.numFalsePositives] + res[:true_negatives] = s[OT.numTrueNegatives] + res[:false_negatives] = s[OT.numFalseNegatives] + res[:sensitivity] = s[OT.truePositiveRate] + res[:specificity] = s[OT.falsePositiveRate] + break end end + res + elsif v[OT.regressionStatistics] { - :nr_predictions => n, - :true_positives => tp, - :false_positives => fp, - :true_negatives => tn, - :false_negatives => fn, - :correct_predictions => 100*(tp+tn).to_f/n, - :weighted_area_under_roc => v[:classification_statistics][:weighted_area_under_roc].to_f, - :sensitivity => tp.to_f/(tp+fn), - :specificity => tn.to_f/(tn+fp), - } - when "regression" - { - :nr_predictions => v[:num_instances] - v[:num_unpredicted], - :r_square => v[:regression_statistics][:r_square], - :root_mean_squared_error => v[:regression_statistics][:root_mean_squared_error], - :mean_absolute_error => v[:regression_statistics][:mean_absolute_error], + :nr_predictions => v[OT.numInstances] - v[OT.numUnpredicted], + :r_square => v[OT.regressionStatistics][OT.rSquare], + :root_mean_squared_error => v[OT.regressionStatistics][OT.rootMeanSquaredError], + :mean_absolute_error => v[OT.regressionStatistics][OT.meanAbsoluteError], } end end + end - end + class CrossvalidationReport + include OpenTox + + # finds CrossvalidationReport via uri, raises error if not found + # @param [String] uri + # @param [String,optional] subjectid + # @return [OpenTox::CrossvalidationReport] + def self.find( uri, subjectid=nil ) + # PENDING load report data? + OpenTox::RestClientWrapper.get(uri,{:subjectid => subjectid}) + CrossvalidationReport.new(uri) + end + + # finds CrossvalidationReport for a particular crossvalidation + # @param [OpenTox::Crossvalidation] + # @param [String,optional] subjectid + # @return [OpenTox::CrossvalidationReport] nil if no report found + def self.find_for_crossvalidation( crossvalidation, subjectid=nil ) + uris = RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"], + "/report/crossvalidation?crossvalidation="+crossvalidation.uri), {:subjectid => subjectid}).chomp.split("\n") + uris.size==0 ? nil : CrossvalidationReport.new(uris[-1]) + end + + # creates a crossvalidation report via crossvalidation + # @param [OpenTox::Crossvalidation] + # @param [String,optional] subjectid + # @param [OpenTox::Task,optional] waiting_task (can be a OpenTox::Subtask as well), progress is updated accordingly + # @return [OpenTox::CrossvalidationReport] + def self.create( crossvalidation, subjectid=nil, waiting_task=nil ) + uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/report/crossvalidation"), + { :validation_uris => crossvalidation.uri, :subjectid => subjectid }, {}, waiting_task ) + CrossvalidationReport.new(uri) + end + end + + class QMRFReport + include OpenTox + + # finds QMRFReport, raises Error if not found + # @param [String] uri + # @param [String,optional] subjectid + # @return [OpenTox::QMRFReport] + def self.find( uri, subjectid=nil ) + # PENDING load crossvalidation data? + OpenTox::RestClientWrapper.get(uri,{:subjectid => subjectid}) + QMRFReport.new(uri) + end + + # finds QMRF report for a particular model + # @param [OpenTox::Crossvalidation] + # @param [String,optional] subjectid + # @return [OpenTox::QMRFReport] nil if no report found + def self.find_for_model( model, subjectid=nil ) + uris = RestClientWrapper.get(File.join(CONFIG[:services]["opentox-validation"], + "/reach_report/qmrf?model="+model.uri), {:subjectid => subjectid}).chomp.split("\n") + uris.size==0 ? nil : QMRFReport.new(uris[-1]) + end + + # creates a qmrf report via model + # @param [OpenTox::Model] + # @param [String,optional] subjectid + # @param [OpenTox::Task,optional] waiting_task (can be a OpenTox::Subtask as well), progress is updated accordingly + # @return [OpenTox::QMRFReport] + def self.create( model, subjectid=nil, waiting_task=nil ) + uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/reach_report/qmrf"), + { :model_uri => model.uri, :subjectid => subjectid }, {}, waiting_task ) + QMRFReport.new(uri) + end + end + end -- cgit v1.2.3