From c945cad0eb0b01fb897b0dbb95d14a9872da5e1c Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 29 Jun 2011 11:19:02 +0200 Subject: do delete policies in background with sleeps, fix cleanup method --- validation/validation_application.rb | 28 ++++++++--------- validation/validation_service.rb | 60 ++++++++++++++++++++++-------------- 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/validation/validation_application.rb b/validation/validation_application.rb index bd55d4c..c9a5649 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -61,17 +61,14 @@ post '/crossvalidation/cleanup/?' do LOGGER.info "crossvalidation cleanup, starting..." content_type "text/uri-list" deleted = [] - #Validation::Crossvalidation.find_like(params).each do |cv| - Validation::Crossvalidation.all( { :finished => false } ).each do |cv| - #num_vals = Validation::Validation.find( :all, :conditions => { :crossvalidation_id => cv.id } ).size - #num_vals = Validation::Validation.all( :crossvalidation_id => cv.id ).size - #if cv.num_folds != num_vals || !cv.finished + Validation::Crossvalidation.all.collect.delete_if{|cv| cv.finished}.each do |cv| + if OpenTox::Authorization.authorized?(cv.crossvalidation_uri,"DELETE",@subjectid) LOGGER.debug "delete cv with id:"+cv.id.to_s+", finished is false" deleted << cv.crossvalidation_uri - #Validation::Crossvalidation.delete(cv.id) cv.subjectid = @subjectid cv.delete_crossvalidation - #end + sleep 1 if AA_SERVER + end end LOGGER.info "crossvalidation cleanup, deleted "+deleted.size.to_s+" cvs" deleted.join("\n")+"\n" @@ -189,8 +186,8 @@ delete '/crossvalidation/:id/?' do # Validation::Crossvalidation.delete(params[:id]) cv = Validation::Crossvalidation.get(params[:id]) - cv.subjectid = @subjectid raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found." unless cv + cv.subjectid = @subjectid cv.delete_crossvalidation end @@ -474,13 +471,14 @@ post '/cleanup/?' do LOGGER.info "validation cleanup, starting..." content_type "text/uri-list" deleted = [] - #Validation::Validation.find( :all, :conditions => { :prediction_dataset_uri => nil } ).each do |val| - Validation::Validation.all( :finished => false ).each do |val| - LOGGER.debug "delete val with id:"+val.id.to_s+", finished is false" - deleted << val.validation_uri - #Validation::Validation.delete(val.id) - val.subjectid = @subjectid - val.delete_validation + Validation::Validation.all.collect.delete_if{|val| val.finished}.each do |val| + if OpenTox::Authorization.authorized?(val.validation_uri,"DELETE",@subjectid) + LOGGER.debug "delete val with id:"+val.id.to_s+", finished is false" + deleted << val.validation_uri + val.subjectid = @subjectid + val.delete_validation + sleep 1 if AA_SERVER + end end LOGGER.info "validation cleanup, deleted "+deleted.size.to_s+" validations" deleted.join("\n")+"\n" diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 77cc694..8dc90e2 100755 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -84,38 +84,45 @@ module Validation if (delete_all) to_delete = [:model_uri, :training_dataset_uri, :test_dataset_uri, :test_target_dataset_uri, :prediction_dataset_uri ] case self.validation_type - when /test_set_validation/ + when "test_set_validation" to_delete -= [ :model_uri, :training_dataset_uri, :test_dataset_uri, :test_target_dataset_uri ] - when /bootstrapping/ + when "bootstrapping" to_delete -= [ :test_target_dataset_uri ] - when /training_test_validation/ + when "training_test_validation" to_delete -= [ :training_dataset_uri, :test_dataset_uri, :test_target_dataset_uri ] - when /training_test_split/ + when "training_test_split" to_delete -= [ :test_target_dataset_uri ] - when /validate_dataset/ + when "validate_datasets" to_delete = [] - when /crossvalidation/ + when "crossvalidation" to_delete -= [ :test_target_dataset_uri ] + when "crossvalidation_statistics" + to_delete = [] else - raise "unknown dataset type" + raise "unknown validation type '"+self.validation_type.to_s+"'" end - to_delete.each do |attr| - uri = self.send(attr) - LOGGER.debug "also deleting "+attr.to_s+" : "+uri.to_s if uri - begin - OpenTox::RestClientWrapper.delete(uri, :subjectid => subjectid) if uri - rescue => ex - LOGGER.warn "could not delete "+uri.to_s+" : "+ex.message.to_s + Thread.new do # do deleting in background to not cause a timeout + to_delete.each do |attr| + uri = self.send(attr) + LOGGER.debug "also deleting "+attr.to_s+" : "+uri.to_s if uri + begin + OpenTox::RestClientWrapper.delete(uri, :subjectid => subjectid) if uri + sleep 1 if AA_SERVER # wait a second to not stress the a&a service too much + rescue => ex + LOGGER.warn "could not delete "+uri.to_s+" : "+ex.message.to_s + end end end end self.delete if (subjectid) - begin - res = OpenTox::Authorization.delete_policies_from_uri(validation_uri, subjectid) - LOGGER.debug "Deleted validation policy: #{res}" - rescue - LOGGER.warn "Policy delete error for validation: #{validation_uri}" + Thread.new do + begin + res = OpenTox::Authorization.delete_policies_from_uri(validation_uri, subjectid) + LOGGER.debug "Deleted validation policy: #{res}" + rescue + LOGGER.warn "Policy delete error for validation: #{validation_uri}" + end end end "Successfully deleted validation "+self.id.to_s+"." @@ -272,12 +279,18 @@ module Validation # deletes a crossvalidation, all validations are deleted as well def delete_crossvalidation - Validation.find(:crossvalidation_id => self.id).each do |v| + validations = Validation.find(:crossvalidation_id => self.id) + Thread.new do # do deleting in background to not cause a timeout + validations.each do |v| v.subjectid = self.subjectid + LOGGER.debug "deleting cv-validation "+v.validation_uri.to_s v.delete_validation + sleep 1 if AA_SERVER # wait a second to not stress the a&a service too much end - self.delete - if (subjectid) + end + self.delete + if (subjectid) + Thread.new do begin res = OpenTox::Authorization.delete_policies_from_uri(crossvalidation_uri, subjectid) LOGGER.debug "Deleted crossvalidation policy: #{res}" @@ -285,7 +298,8 @@ module Validation LOGGER.warn "Policy delete error for crossvalidation: #{crossvalidation_uri}" end end - "Successfully deleted crossvalidation "+self.id.to_s+"." + end + "Successfully deleted crossvalidation "+self.id.to_s+"." end # creates the cv folds -- cgit v1.2.3