From 9fe1f6870cfd12c34eb4efef8f4e199e8324c1af Mon Sep 17 00:00:00 2001 From: Christoph Helma Date: Sun, 19 Feb 2012 16:03:10 +0000 Subject: task handling fixed for http codes > 202 --- .gitignore | 13 ++---- Rakefile | 21 --------- lib/authorization.rb | 30 ++++++------- lib/error.rb | 3 +- lib/opentox-client.rb | 5 ++- lib/opentox.rb | 117 +++++++++++++++++++++---------------------------- lib/spork.rb | 83 ----------------------------------- lib/task.rb | 68 +++++++++++++++++++++++----- opentox-client.gemspec | 1 + test/task.rb | 38 ++++++++++++---- 10 files changed, 163 insertions(+), 216 deletions(-) delete mode 100644 lib/spork.rb diff --git a/.gitignore b/.gitignore index 75924e1..4040c6c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,4 @@ -*.sw? -.DS_Store -coverage -rdoc -pkg -.yardoc -doc -mysql-bak.rb -*~ +*.gem +.bundle +Gemfile.lock +pkg/* diff --git a/Rakefile b/Rakefile index 980b5ba..5bcc76a 100644 --- a/Rakefile +++ b/Rakefile @@ -1,26 +1,5 @@ require "bundler/gem_tasks" -=begin - require 'jeweler' - Jeweler::Tasks.new do |gem| - gem.name = "opentox-client" - gem.summary = %Q{Ruby wrapper for the OpenTox REST API} - gem.description = %Q{Ruby wrapper for the OpenTox REST API (http://www.opentox.org)} - gem.email = "helma@in-silico.ch" - gem.homepage = "http://github.com/opentox/opentox-ruby-minimal" - gem.authors = ["Christoph Helma, Martin Guetlein, Andreas Maunz, Micha Rautenberg, David Vorgrimmler"] - # dependencies with versions - gem.add_dependency "rest-client" - gem.add_dependency "rdf" - gem.add_dependency "rdf-raptor" - gem.add_development_dependency 'jeweler' - gem.files = FileList["[A-Z]*", "{bin,generators,lib,test}/**/*", 'lib/jeweler/templates/.gitignore'] - end - Jeweler::GemcutterTasks.new -rescue LoadError - puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler" -=end - require 'rake/testtask' Rake::TestTask.new do |t| t.libs << 'lib' diff --git a/lib/authorization.rb b/lib/authorization.rb index d447f88..1938814 100644 --- a/lib/authorization.rb +++ b/lib/authorization.rb @@ -41,8 +41,8 @@ module OpenTox xml = get_xml(uri) ret = false ret = Authorization.create_policy(xml, @subjectid) - LOGGER.debug "Policy send with subjectid: #{@subjectid}" - LOGGER.warn "Not created Policy is: #{xml}" if !ret + @@logger.debug "Policy send with subjectid: #{@subjectid}" + @@logger.warn "Not created Policy is: #{xml}" if !ret ret end @@ -200,7 +200,7 @@ module OpenTox def self.create_policy(policy, subjectid) begin resource = RestClient::Resource.new("#{AA_SERVER}/Pol/opensso-pol") - LOGGER.debug "OpenTox::Authorization.create_policy policy: #{policy[168,43]} with token:" + subjectid.to_s + " length: " + subjectid.length.to_s + @@logger.debug "OpenTox::Authorization.create_policy policy: #{policy[168,43]} with token:" + subjectid.to_s + " length: " + subjectid.length.to_s return true if resource.post(policy, :subjectid => subjectid, :content_type => "application/xml") rescue return false @@ -213,7 +213,7 @@ module OpenTox def self.delete_policy(policy, subjectid) begin resource = RestClient::Resource.new("#{AA_SERVER}/pol") - LOGGER.debug "OpenTox::Authorization.delete_policy policy: #{policy} with token: #{subjectid}" + @@logger.debug "OpenTox::Authorization.delete_policy policy: #{policy} with token: #{subjectid}" return true if resource.delete(:subjectid => subjectid, :id => policy) rescue return nil @@ -279,7 +279,7 @@ module OpenTox return true if !AA_SERVER aa = Authorization::AA.new(subjectid) ret = aa.send(uri) - LOGGER.debug "OpenTox::Authorization send policy for URI: #{uri} | subjectid: #{subjectid} - policy created: #{ret}" + @@logger.debug "OpenTox::Authorization send policy for URI: #{uri} | subjectid: #{subjectid} - policy created: #{ret}" ret end @@ -291,7 +291,7 @@ module OpenTox if policies policies.each do |policy| ret = delete_policy(policy, subjectid) - LOGGER.debug "OpenTox::Authorization delete policy: #{policy} - with result: #{ret}" + @@logger.debug "OpenTox::Authorization delete policy: #{policy} - with result: #{ret}" end end return true @@ -304,11 +304,11 @@ module OpenTox def self.check_policy(uri, subjectid) return true unless uri and subjectid token_valid = OpenTox::Authorization.is_token_valid(subjectid) - LOGGER.debug "OpenTox::Authorization.check_policy with uri: #{uri}, subjectid: #{subjectid} is valid: #{token_valid}" + @@logger.debug "OpenTox::Authorization.check_policy with uri: #{uri}, subjectid: #{subjectid} is valid: #{token_valid}" # check if subjectid is valid unless token_valid # abort if invalid - LOGGER.error "OpenTox::Authorization.check_policy, subjectid NOT valid: #{subjectid}" + @@logger.error "OpenTox::Authorization.check_policy, subjectid NOT valid: #{subjectid}" return false end @@ -320,7 +320,7 @@ module OpenTox if authorize(uri, "POST", subjectid) true else - LOGGER.error "OpenTox::Authorization.check_policy, already exists, but no POST-authorization with subjectid: #{subjectid}" + @@logger.error "OpenTox::Authorization.check_policy, already exists, but no POST-authorization with subjectid: #{subjectid}" false end end @@ -338,25 +338,25 @@ module OpenTox # @return [Boolean] true if access granted, else otherwise def self.authorized?(uri, request_method, subjectid) if CONFIG[:authorization][:free_request].include?(request_method) - #LOGGER.debug "authorized? >>true<< (request is free), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" + #@@logger.debug "authorized? >>true<< (request is free), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" true elsif OpenTox::Authorization.free_uri?(uri, request_method) - #LOGGER.debug "authorized? >>true<< (uris is free_uri), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" + #@@logger.debug "authorized? >>true<< (uris is free_uri), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" true elsif CONFIG[:authorization][:authenticate_request].include?(request_method) ret = OpenTox::Authorization.is_token_valid(subjectid) - LOGGER.debug "authorized? >>#{ret}<< (token is in/valid), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" unless ret + @@logger.debug "authorized? >>#{ret}<< (token is in/valid), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" unless ret ret elsif OpenTox::Authorization.authorize_exception?(uri, request_method) ret = OpenTox::Authorization.is_token_valid(subjectid) - LOGGER.debug "authorized? >>#{ret}<< (uris is authorize exception, token is in/valid), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" unless ret + @@logger.debug "authorized? >>#{ret}<< (uris is authorize exception, token is in/valid), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" unless ret ret elsif CONFIG[:authorization][:authorize_request].include?(request_method) ret = OpenTox::Authorization.authorize(uri, request_method, subjectid) - LOGGER.debug "authorized? >>#{ret}<< (uri (not) authorized), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" unless ret + @@logger.debug "authorized? >>#{ret}<< (uri (not) authorized), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" unless ret ret else - LOGGER.error "invalid request/uri method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" + @@logger.error "invalid request/uri method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}" false end end diff --git a/lib/error.rb b/lib/error.rb index b92f2a4..64cc4eb 100644 --- a/lib/error.rb +++ b/lib/error.rb @@ -1,4 +1,3 @@ - # adding additional fields to Exception class to format errors according to OT-API class Exception attr_accessor :errorCause @@ -50,7 +49,7 @@ module OpenTox # @param [String] actor, URI of the call that cause the error def self.create( error, actor ) rest_params = error.rest_params if error.is_a?(OpenTox::RestCallError) and error.rest_params - backtrace = error.backtrace.short_backtrace if CONFIG[:backtrace] + backtrace = error.backtrace.short_backtrace #if CONFIG[:backtrace] ErrorReport.new( error.http_code, error.class.to_s, error.message, actor, error.errorCause, rest_params, backtrace ) end diff --git a/lib/opentox-client.rb b/lib/opentox-client.rb index 0bd048c..c5c701b 100644 --- a/lib/opentox-client.rb +++ b/lib/opentox-client.rb @@ -1,9 +1,12 @@ -require "opentox-client/version" require 'rubygems' require "bundler/setup" require 'rdf' require 'rdf/raptor' require "rest-client" +require 'uri' +require 'yaml' +require File.join(File.dirname(__FILE__),"error.rb") +require File.join(File.dirname(__FILE__),"logger.rb") require File.join(File.dirname(__FILE__),"opentox.rb") require File.join(File.dirname(__FILE__),"task.rb") require File.join(File.dirname(__FILE__),"compound.rb") diff --git a/lib/opentox.rb b/lib/opentox.rb index 51bc17a..ab5c95f 100644 --- a/lib/opentox.rb +++ b/lib/opentox.rb @@ -1,5 +1,4 @@ #TODO: switch services to 1.2 -#TODO: error handling RDF::OT = RDF::Vocabulary.new 'http://www.opentox.org/api/1.2#' RDF::OT1 = RDF::Vocabulary.new 'http://www.opentox.org/api/1.1#' RDF::OTA = RDF::Vocabulary.new 'http://www.opentox.org/algorithmTypes.owl#' @@ -11,83 +10,62 @@ RestClient.add_before_execution_proc do |req, params| end class String - def to_object - # TODO: fix, this is unsafe - self =~ /dataset/ ? uri = File.join(self.chomp,"metadata") : uri = self.chomp - raise "#{uri} is not a valid URI." unless RDF::URI.new(uri).uri? - RDF::Reader.open(uri) do |reader| - reader.each_statement do |statement| - if statement.predicate == RDF.type and statement.subject == uri - klass = "OpenTox::#{statement.object.to_s.split("#").last}" - object = eval "#{klass}.new \"#{uri}\"" - end - end - end - # fallback: guess class from uri - # TODO: fix services and remove - unless object - case uri - when /compound/ - object = OpenTox::Compound.new uri - when /feature/ - object = OpenTox::Feature.new uri - when /dataset/ - object = OpenTox::Dataset.new uri.sub(/\/metadata/,'') - when /algorithm/ - object = OpenTox::Algorithm.new uri - when /model/ - object = OpenTox::Model.new uri - when /validation/ - object = OpenTox::Validation.new uri - when /task/ - object = OpenTox::Task.new uri - else - raise "Class for #{uri} not found." - end - end - if object.class == Task # wait for tasks - object.wait_for_completion - object = object.result_uri.to_s.to_object - end - object - end - -=begin - def object_from_uri + def to_object # TODO: fix, this is unsafe self =~ /dataset/ ? uri = File.join(self.chomp,"metadata") : uri = self.chomp + raise "#{uri} is not a valid URI." unless RDF::URI.new(uri).uri? RDF::Reader.open(uri) do |reader| reader.each_statement do |statement| if statement.predicate == RDF.type and statement.subject == uri klass = "OpenTox::#{statement.object.to_s.split("#").last}" - return eval "#{klass}.new \"#{uri}\"" + object = eval "#{klass}.new \"#{uri}\"" end end end - # guess class from uri + # fallback: guess class from uri # TODO: fix services and remove - case uri - when /compound/ - return OpenTox::Compound.new uri - when /feature/ - return OpenTox::Feature.new uri - when /dataset/ - return OpenTox::Dataset.new uri.sub(/\/metadata/,'') - when /algorithm/ - return OpenTox::Algorithm.new uri - when /model/ - return OpenTox::Model.new uri - when /validation/ - return OpenTox::Validation.new uri - when /task/ - return OpenTox::Task.new uri - else - raise "Class for #{uri} not found." + unless object + case uri + when /compound/ + object = OpenTox::Compound.new uri + when /feature/ + object = OpenTox::Feature.new uri + when /dataset/ + object = OpenTox::Dataset.new uri.sub(/\/metadata/,'') + when /algorithm/ + object = OpenTox::Algorithm.new uri + when /model/ + object = OpenTox::Model.new uri + when /validation/ + object = OpenTox::Validation.new uri + when /task/ + object = OpenTox::Task.new uri + else + raise "Class for #{uri} not found." + end + end + if object.class == Task # wait for tasks + object.wait_for_completion + object = object.result_uri.to_s.to_object + end + object + end + + def uri? + begin + Net::HTTP.get_response(URI.parse(self)) + true + rescue + false end end -=end end + +# defaults to stderr, may be changed to file output +$logger = OTLogger.new(STDERR) # no rotation +$logger.level = Logger::DEBUG + module OpenTox attr_accessor :subjectid, :uri, :response @@ -103,10 +81,15 @@ module OpenTox def metadata reload=true if reload @metadata = {} - RDF::Reader.open(@uri) do |reader| - reader.each_statement do |statement| - @metadata[statement.predicate] = statement.object if statement.subject == @uri + begin + RDF::Reader.open(@uri) do |reader| + reader.each_statement do |statement| + @metadata[statement.predicate] = statement.object if statement.subject == @uri + end end + rescue + $logger.error "Cannot read RDF metadata from #{@uri}: #{$!}.\n#{$!.backtrace.join("\n")}" + raise end end @metadata diff --git a/lib/spork.rb b/lib/spork.rb deleted file mode 100644 index c77b5b5..0000000 --- a/lib/spork.rb +++ /dev/null @@ -1,83 +0,0 @@ -# A way to cleanly handle process forking in Sinatra when using Passenger, aka "sporking some code". -# This will allow you to properly execute some code asynchronously, which otherwise does not work correctly. -# -# Written by Ron Evans -# More info at http://deadprogrammersociety.com -# -# Mostly lifted from the Spawn plugin for Rails (http://github.com/tra/spawn) -# but with all of the Rails stuff removed.... cause you are using Sinatra. If you are using Rails, Spawn is -# what you need. If you are using something else besides Sinatra that is Rack-based under Passenger, and you are having trouble with -# asynch processing, let me know if spork helped you. -# -module Spork - # things to close in child process - @@resources = [] - def self.resources - @@resources - end - - # set the resource to disconnect from in the child process (when forking) - def self.resource_to_close(resource) - @@resources << resource - end - - # close all the resources added by calls to resource_to_close - def self.close_resources - @@resources.each do |resource| - resource.close if resource && resource.respond_to?(:close) && !resource.closed? - end - @@resources = [] - end - - # actually perform the fork... er, spork - # valid options are: - # :priority => to set the process priority of the child - # :logger => a logger object to use from the child - # :no_detach => true if you want to keep the child process under the parent control. usually you do NOT want this - def self.spork(options={}) - logger = options[:logger] - logger.debug "spork> parent PID = #{Process.pid}" if logger - - child = fork do - begin - start = Time.now - logger.debug "spork> child PID = #{Process.pid}" if logger - - # set the nice priority if needed - Process.setpriority(Process::PRIO_PROCESS, 0, options[:priority]) if options[:priority] - - # disconnect from the rack - Spork.close_resources - - # run the block of code that takes so long - yield - - rescue => ex - #raise ex - logger.error "spork> Exception in child[#{Process.pid}] - #{ex.class}: #{ex.message}" if logger - ensure - logger.info "spork> child[#{Process.pid}] took #{Time.now - start} sec" if logger - # this form of exit doesn't call at_exit handlers - exit!(0) - end - end - - # detach from child process (parent may still wait for detached process if they wish) - Process.detach(child) unless options[:no_detach] - - return child - end - -end - -# Patch to work with passenger -if defined? Passenger::Rack::RequestHandler - class Passenger::Rack::RequestHandler - alias_method :orig_process_request, :process_request - def process_request(env, input, output) - Spork.resource_to_close(input) - Spork.resource_to_close(output) - orig_process_request(env, input, output) - end - end -end diff --git a/lib/task.rb b/lib/task.rb index 50616d7..52d4a30 100644 --- a/lib/task.rb +++ b/lib/task.rb @@ -1,28 +1,47 @@ -require File.join(File.dirname(__FILE__),'spork') +require File.join(File.dirname(__FILE__),'error') DEFAULT_TASK_MAX_DURATION = 36000 module OpenTox # Class for handling asynchronous tasks class Task + attr_accessor :pid def self.create service_uri, params={} task = Task.new RestClient.post(service_uri,params).chomp - pid = Spork.spork do + pid = fork do begin - task.completed yield - rescue => error - task.error error + result_uri = yield + if result_uri.uri? + task.completed result_uri + else + raise "#{result_uri} is not a valid URI" + end + rescue + # TODO add service URI to Kernel.raise + # serialize error and send to task service + #task.error $! + task.error $! + raise end end + Process.detach(pid) task.pid = pid task end + def kill + begin + Process.kill(9,pid) + rescue + end + end + def description metadata[RDF::DC.description] end def cancel + kill RestClient.put(File.join(@uri,'Cancelled'),{}) end @@ -30,19 +49,24 @@ module OpenTox RestClient.put(File.join(@uri,'Completed'),{:resultURI => uri}) end - def error(error) - RestClient.put(File.join(@uri,'Error'),{:errorReport => OpenTox::Error.new(error)}) + def error error + $logger.error self if $logger + kill + report = ErrorReport.create(error,"http://localhost") + RestClient.put(File.join(@uri,'Error'),{:errorReport => report}) + #RestClient.put(File.join(@uri,'Error'),{:message => error, :backtrace => error.backtrace}) end # waits for a task, unless time exceeds or state is no longer running - # @param [optional,Numeric] dur seconds pausing before cheking again for completion + # @param [optional,Numeric] dur seconds pausing before checking again for completion def wait_for_completion(dur=0.3) due_to_time = Time.new + DEFAULT_TASK_MAX_DURATION - while self.running? + while running? sleep dur raise "max wait time exceeded ("+DEFAULT_TASK_MAX_DURATION.to_s+"sec), task: '"+@uri.to_s+"'" if (Time.new > due_to_time) end end + end def method_missing(method,*args) @@ -55,13 +79,37 @@ module OpenTox when /\?/ return hasStatus == method.sub(/\?/,'').capitalize else - return metadata[RDF::OT[method]].to_s + response = metadata[RDF::OT[method]].to_s + response = metadata[RDF::OT1[method]].to_s #if response.empty? # API 1.1 compatibility + if response.empty? + $logger.error "No #{method} metadata for #{@uri} " + raise "No #{method} metadata for #{@uri} " + end + return response end rescue + $logger.error "Unknown #{self.class} method #{method}" super end end + # override to read all error codes + def metadata reload=true + if reload + @metadata = {} + # ignore error codes from Task services (may contain eg 500 which causes exceptions in RestClient and RDF::Reader + RestClient.get(@uri) do |response, request, result, &block| + $logger.warn "#{@uri} returned #{result}" unless response.code == 200 or response.code == 202 + RDF::Reader.for(:rdfxml).new(response) do |reader| + reader.each_statement do |statement| + @metadata[statement.predicate] = statement.object if statement.subject == @uri + end + end + end + end + @metadata + end + #TODO: subtasks end diff --git a/opentox-client.gemspec b/opentox-client.gemspec index 5477004..7ba3ab0 100644 --- a/opentox-client.gemspec +++ b/opentox-client.gemspec @@ -20,6 +20,7 @@ Gem::Specification.new do |s| # specify any dependencies here; for example: # s.add_development_dependency "rspec" + s.add_runtime_dependency "bundler" s.add_runtime_dependency "rest-client" s.add_runtime_dependency "rdf" s.add_runtime_dependency "rdf-raptor" diff --git a/test/task.rb b/test/task.rb index 41316d1..b217ec2 100644 --- a/test/task.rb +++ b/test/task.rb @@ -3,32 +3,54 @@ $LOAD_PATH << File.join(File.dirname(__FILE__),'..','lib') require File.join File.dirname(__FILE__),'..','lib','opentox-client.rb' #require "./validate-owl.rb" -TASK_SERVICE_URI = "http://ot-dev.in-silico.ch/task" +#TASK_SERVICE_URI = "http://ot-dev.in-silico.ch/task" +TASK_SERVICE_URI = "http://ot-test.in-silico.ch/task" +#TASK_SERVICE_URI = "https://ambit.uni-plovdiv.bg:8443/ambit2/task" #not compatible class TaskTest < Test::Unit::TestCase - def setup - end - def teardown + def test_all + t = OpenTox::Task.all(TASK_SERVICE_URI) + assert_equal Array, t.class + assert_equal RDF::OT1.Task, t.last.metadata[RDF.type] end def test_create_and_complete - task = OpenTox::Task.create TASK_SERVICE_URI do + task = OpenTox::Task.create TASK_SERVICE_URI, :description => "test" do sleep 1 - "http://test.org" + TASK_SERVICE_URI end assert_equal "Running", task.hasStatus task.wait_for_completion assert_equal "Completed", task.hasStatus - assert_equal "http://test.org", task.resultURI + assert_equal TASK_SERVICE_URI, task.resultURI end - def test_rdf task = OpenTox::Task.all(TASK_SERVICE_URI).last assert_equal OpenTox::Task, task.class #validate_owl(task.uri) end + def test_create_and_fail + task = OpenTox::Task.create TASK_SERVICE_URI, :description => "test failure", :creator => "http://test.org/fake_creator" do + sleep 1 + raise "an error occured" + end + assert_equal "Running", task.hasStatus + task.wait_for_completion + assert_equal "Error", task.hasStatus + end + + def test_wrong_result_uri + task = OpenTox::Task.create TASK_SERVICE_URI, :description => "test wrong result uri", :creator => "http://test.org/fake_creator" do + sleep 1 + "Asasadasd" + end + assert_equal "Running", task.hasStatus + task.wait_for_completion + assert_equal "Error", task.hasStatus + end + end -- cgit v1.2.3