diff options
| -rw-r--r-- | EXAMPLES | 119 | ||||
| -rw-r--r-- | application.rb | 92 | ||||
| -rw-r--r-- | validation/validation_application.rb | 8 | ||||
| -rw-r--r-- | validation/validation_format.rb | 11 | ||||
| -rw-r--r-- | validation/validation_service.rb | 19 | ||||
| -rw-r--r-- | validation/validation_test.rb | 56 |
6 files changed, 220 insertions, 85 deletions
@@ -1,79 +1,142 @@ API-Definition -==================================== +=================================================================== see http://opentox.org/dev/apis/api-1.1/Validation API-Examples -==================================== +=================================================================== how to ------------------------------------- +------------------------------------------------------------------- the examples listed below can be performed with the command line tool curl get list of available validations ------------------------------------- +------------------------------------------------------------------- >>> curl <validation_service>/validations result example (accept-header: application/rdf-xml) -<<< TODO +<<< not yet supported result example (accept-header: text/uri-list) <<< <validation_service>/validation/<validation_id> <<< <validation_service>/validation/validation_id_i <<< <validation_service>/validation/validation_id_j + + +get validation +------------------------------------------------------------------- + +>>> curl <validation_service>/validation/<validation_id> + +supported accept-headers: "text/x-yaml" and "application/rdf-xml" + validate model on test-dateset ------------------------------------- +------------------------------------------------------------------- ->>> curl -X POST -d model_uri="<model_service>/model/<model_id>" \ - -d dataset_uri="<dataset_service>/dataset/<dataset_id>" \ - <validation_service>/validation +>>> curl -X POST -d model_uri="<model_uri>" \ + -d test_dataset_uri="<test_dataset_uri>" \ + -d prediction_feature="<prediction_feature>" \ + <validation_service>/validation result example (accept-header: application/rdf-xml) -<<< TODO +<<< not yet supported result example (accept-header: text/uri-list) -<<< <validation_service>/validation/<id> +<<< <validation_service>/validation/id_i validate an algorithm on a training- and test-dataset ------------------------------------- +------------------------------------------------------------------- + +>>> curl -X POST -d algorithm_uri="<algorithm_uri>" \ + -d training_dataset_uri="<training_dataset_uri>" \ + -d test_dataset_uri="<test_dataset_uri>" \ + -d prediction_feature="<prediction_feature>" \ + -d algorithm_params="<algorithm_params>" \ + <validation_service>/validation ->>> curl -X POST -d algorithm_uri="<model_service>/model/<model_id>" \ - -d training_dataset_uri="<dataset_service>/dataset/<dataset_id>" \ - -d test_dataset_uri="<dataset_service>/dataset/<dataset_id>" \ - -d prediction_feature="<prediction_feature>" \ - <validation_service>/validation +optional params: +algorithm_params, default is empty result example (accept-header: application/rdf-xml) -<<< TODO +<<< not yet supported result example (accept-header: text/uri-list) -<<< <validation_service>/validation/<id> +<<< <validation_service>/validation/id_i validate an algorithm on a dataset via training-test-split ------------------------------------- +------------------------------------------------------------------- ->>> curl -X POST -d algorithm_uri="<model_service>/model/<model_id>" \ - -d dataset_uri="<dataset_service>/dataset/<dataset_id>" \ - -d prediction_feature="<prediction_feature>" \ - -d split_ratio=0.9 \ - -d random_seed=2 \ - <validation_service>/validation/training-test-split +>>> curl -X POST -d algorithm_uri="<algorithm_uri>" \ + -d dataset_uri="<dataset_uri>" \ + -d prediction_feature="<prediction_feature>" \ + -d algorithm_params="<algorithm_params>" \ + -d split_ratio=0.9 \ + -d random_seed=2 \ + <validation_service>/validation/training_test_split optional params: +algorithm_params, default is empty split_ratio, default is 0.67 random_seed, default is 1 result example (accept-header: application/rdf-xml) -<<< TODO +<<< not yet supported + +result example (accept-header: text/uri-list) +<<< <validation_service>/validation/id_i + + +get list of available crossvalidations +------------------------------------------------------------------- + +>>> curl <validation_service>/crossvalidations + +result example (accept-header: application/rdf-xml) +<<< not yet supported + +result example (accept-header: text/uri-list) +<<< <validation_service>/crossvalidation/<crossvalidation_id> +<<< <validation_service>/crossvalidation/crossvalidation_id_i +<<< <validation_service>/crossvalidation/crossvalidation_id_j + + +get crossvalidation +------------------------------------------------------------------- + +>>> curl <validation_service>/crossvalidation/<crossvalidation_id> + +supported accept-headers: "text/x-yaml" and "application/rdf-xml" + + +validate an algorithm on a dataset via crossvalidation +------------------------------------------------------------------- + +>>> curl -X POST -d algorithm_uri="<algorithm_uri>" \ + -d dataset_uri="<dataset_uri>" \ + -d prediction_feature="<prediction_feature>" \ + -d algorithm_params="<algorithm_params>" \ + -d num_folds=5 \ + -d random_seed=2 \ + -d stratified=false \ + <validation_service>/crossvalidation + +optional params: +algorithm_params, default is empty +num_folds, default is 10 +random_seed, default is 1 +stratified, default is false + +result example (accept-header: application/rdf-xml) +<<< not yet supported result example (accept-header: text/uri-list) -<<< <validation_service>/validation/<id> +<<< <validation_service>/crossvalidation/id_i diff --git a/application.rb b/application.rb index fa444be..1c7106f 100644 --- a/application.rb +++ b/application.rb @@ -8,30 +8,86 @@ require "report/report_application.rb" end get '/examples/?' do - - transform_example + content_type "text/plain" + Demo.transform_example +end + +get '/prepare_examples/?' do + Demo.prepare_example_resources + "done" end private -def transform_example - file = File.new("EXAMPLES", "r") - res = "" - while (line = file.gets) - res += line - end - file.close +class Demo - sub = { "validation_service" => url_for("", :full), - #"validation_id" => "??", - #"model_service" => "??", - #"model_id" => "??", - #"dataset_service" => "??", - "dataset_id" => "Hamster Carcenogenicity"} + @@file=File.new("data/hamster_carcinogenicity.owl","r") + @@model=File.join @@config[:services]["opentox-model"],"1" + @@feature="http://www.epa.gov/NCCT/dsstox/CentralFieldDef.html#ActivityOutcome_CPDBAS_Hamster" + @@alg = File.join @@config[:services]["opentox-algorithm"],"lazar" + @@alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + @@data=File.join @@config[:services]["opentox-dataset"],"1" + @@train_data=File.join @@config[:services]["opentox-dataset"],"2" + @@test_data=File.join @@config[:services]["opentox-dataset"],"3" - sub.each do |k,v| - res.gsub!(/<#{k}>/,v) + def self.transform_example + + file = File.new("EXAMPLES", "r") + res = "" + while (line = file.gets) + res += line + end + file.close + + sub = { "validation_service" => @@config[:services]["opentox-validation"], + "validation_id" => "1", + "model_uri" => @@model, + "dataset_uri" => @@data, + "training_dataset_uri" => @@train_data, + "test_dataset_uri" => @@test_data, + "prediction_feature" => @@feature, + "algorithm_uri" => @@alg, + "algorithm_params" => @@alg_params, + "crossvalidation_id" => "1",} + + sub.each do |k,v| + res.gsub!(/<#{k}>/,v) + end + res + end + + def self.delete_all(uri_list_service) + uri_list = RestClient.get(uri_list_service) + uri_list.split("\n").each do |uri| + RestClient.delete(uri) + end end - res + def self.prepare_example_resources + + delete_all(@@config[:services]["opentox-dataset"]) + data = File.read(@@file.path) + data_uri = RestClient.post @@config[:services]["opentox-dataset"], data, :content_type => "application/rdf+xml" + puts "uploaded dataset "+data_uri + raise "failed to prepare demo" unless data_uri==@@data + + Lib::Validation.auto_migrate! + delete_all(@@config[:services]["opentox-model"]) + vali_uri = RestClient.post File.join(@@config[:services]["opentox-validation"],'/validation/training_test_split'), { :dataset_uri => data_uri, + :algorithm_uri => @@alg, + :prediction_feature => @@feature, + :algorithm_params => @@alg_params } + puts "created validation via training test split "+vali_uri + raise "failed to prepare demo" unless vali_uri==File.join(@@config[:services]["opentox-validation"],'/validation/1') + + Lib::Crossvalidation.auto_migrate! + cv_uri = RestClient.post File.join(@@config[:services]["opentox-validation"],'/crossvalidation'), { :dataset_uri => data_uri, + :algorithm_uri => @@alg, + :prediction_feature => @@feature, + :algorithm_params => @@alg_params, + :num_folds => 5, :stratified => false } + puts "created crossvalidation "+cv_uri + raise "failed to prepare demo" unless cv_uri==File.join(@@config[:services]["opentox-validation"],'/crossvalidation/1') + + end end diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 42431b4..05909c5 100644 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -67,7 +67,7 @@ post '/crossvalidation/?' do [ :num_folds, :random_seed, :stratified ].each{ |sym| cv_params[sym] = params[sym] if params[sym] } cv = Crossvalidation.new cv_params cv.create_cv_datasets( params[:prediction_feature] ) - cv.perform_cv( params[:feature_generation_uri]) + cv.perform_cv( params[:algorithm_params]) cv.uri end @@ -78,7 +78,7 @@ end get '/validation/:id' do LOGGER.info "get validation with id "+params[:id].to_s+" '"+request.env['HTTP_ACCEPT'].to_s+"'" - halt 404, "Validation #{params[:id]} not found." unless validation = Validation.get(params[:id]) + halt 404, "Validation '#{params[:id]}' not found." unless validation = Validation.get(params[:id]) case request.env['HTTP_ACCEPT'].to_s when "application/rdf+xml" @@ -104,7 +104,7 @@ post '/validation/?' do v = Validation.new :training_dataset_uri => params[:training_dataset_uri], :test_dataset_uri => params[:test_dataset_uri], :prediction_feature => params[:prediction_feature] - v.validate_algorithm( params[:algorithm_uri], params[:feature_generation_uri]) + v.validate_algorithm( params[:algorithm_uri], params[:algorithm_params]) else halt 400, "illegal parameter combination for validation, use either\n"+ "* model_uri, test_dataset_uri, prediction_feature\n"+ @@ -125,7 +125,7 @@ post '/validation/training_test_split' do v = Validation.new :training_dataset_uri => params[:training_dataset_uri], :test_dataset_uri => params[:test_dataset_uri], :prediction_feature => params[:prediction_feature] - v.validate_algorithm( params[:algorithm_uri], params[:feature_generation_uri]) + v.validate_algorithm( params[:algorithm_uri], params[:algorithm_params]) v.uri end diff --git a/validation/validation_format.rb b/validation/validation_format.rb index c5acef9..b953a9b 100644 --- a/validation/validation_format.rb +++ b/validation/validation_format.rb @@ -71,6 +71,12 @@ class Crossvalidation < Lib::Crossvalidation def get_content h = {} Lib::CROSS_VAL_PROPS.each{|p| h[p] = self.send(p)} + + v = [] + Validation.all(:crossvalidation_id => self.id).each do |val| + v.push({ :validation_uri => val.uri.to_s }) + end + h[:validations] = v h end @@ -186,8 +192,9 @@ class CrossvalidationToRDF < ContentToRDF def initialize() @literals = [ :stratified, :num_folds, :random_seed ] - @object_properties = { :dataset_uri => OT['crossvalidationDataset'], :algorithm_uri => OT['corssvalidationAlgorithm'] } - @classes = { } + @object_properties = { :dataset_uri => OT['crossvalidationDataset'], :algorithm_uri => OT['crossvalidationAlgorithm'], + :validation_uri => OT['crossvalidationValidation'], :validations => OT['crossvalidationValidations'] } + @classes = { :validations => OT['CrossvalidationValidations'] } end end diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 3fec7d3..4f738fb 100644 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -32,7 +32,6 @@ end class Validation < Lib::Validation - #include OpenTox::Owl # constructs a validation object, sets id und uri def initialize( params={} ) @@ -61,11 +60,19 @@ class Validation < Lib::Validation end # validates an algorithm by building a model and validating this model - # PENDING: so far, :feature_generation_uri is used to construct a second dataset (first is training-dataset) - def validate_algorithm( algorithm_uri, feature_generation_uri=nil ) + def validate_algorithm( algorithm_uri, algorithm_params=nil ) + + $sinatra.halt 404, "no algorithm uri: '"+algorithm_uri+"'" if algorithm_uri==nil or algorithm_uri.to_s.size<1 params = { :dataset_uri => @training_dataset_uri, :feature_uri => @prediction_feature } - params[:feature_generation_uri] = feature_generation_uri if feature_generation_uri + if (algorithm_params!=nil) + algorithm_params.split(";").each do |alg_params| + alg_param = alg_params.split("=") + #puts "param "+alg_param.to_s + $sinatra.halt 404, "invalid algorithm param: '"+alg_params.to_s+"'" unless alg_param.size==2 or alg_param[0].to_s.size<1 or alg_param[1].to_s.size<1 + params[alg_param[0].to_sym] = alg_param[1] + end + end LOGGER.debug "building model '"+algorithm_uri.to_s+"' "+params.inspect model_uri = RestClient.post algorithm_uri,params @@ -136,11 +143,11 @@ class Crossvalidation < Lib::Crossvalidation end # executes the cross-validation (build models and validates them) - def perform_cv ( feature_generation_uri=nil ) + def perform_cv ( algorithm_params=nil ) LOGGER.debug "perform cv validations" Validation.all( :crossvalidation_id => id ).each do |v| - v.validate_algorithm( @algorithm_uri, feature_generation_uri ) + v.validate_algorithm( @algorithm_uri, algorithm_params ) #break end end diff --git a/validation/validation_test.rb b/validation/validation_test.rb index 1c4a0b8..350e973 100644 --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -80,7 +80,7 @@ class ValidationTest < Test::Unit::TestCase # # num_folds = 9 # post '/crossvalidation', { :dataset_uri => data_uri, :algorithm_uri => WS_CLASS_ALG, :prediction_feature => FEATURE_URI, -# :feature_generation_uri => WS_FEATURE_ALG, :num_folds => num_folds, :random_seed => 2 } +# :algorithm_params => "feature_generation_uri="+WS_FEATURE_ALG, :num_folds => num_folds, :random_seed => 2 } # # puts "crossvalidation: "+last_response.body # assert last_response.ok? @@ -161,34 +161,36 @@ class ValidationTest < Test::Unit::TestCase # end # end - def test_split - begin - data_uri = upload_data(WS_DATA, FILE) - #data_uri = "http://ot.dataset.de/199" #bbrc - #data_uri = "http://ot.dataset.de/67" #hamster - - #data_uri=WS_DATA+"/"+DATA - post '/validation/training_test_split', { :dataset_uri => data_uri, :algorithm_uri => WS_CLASS_ALG, :prediction_feature => FEATURE_URI, - :feature_generation_uri => WS_FEATURE_ALG, :split_ratio=>0.8, :random_seed=>5} - verify_validation - ensure - delete_resources - end - end - -# def test_nothing -# #puts "testing nothing" -# -# #get '/validation/350',nil,'HTTP_ACCEPT' => "application/rdf+xml" -# #get '/validation/350',nil,'HTTP_ACCEPT' => "text/x-yaml" -# #get '/validation-test' -# -# get '/crossvalidation/37',nil,'HTTP_ACCEPT' => "application/rdf+xml" -# #get '/crossvalidation/37',nil,'HTTP_ACCEPT' => "text/x-yaml" -# -# puts last_response.body +# def test_split +# begin +# data_uri = upload_data(WS_DATA, FILE) +# #data_uri = "http://ot.dataset.de/199" #bbrc +# #data_uri = "http://ot.dataset.de/67" #hamster +# +# #data_uri=WS_DATA+"/"+DATA +# post '/validation/training_test_split', { :dataset_uri => data_uri, :algorithm_uri => WS_CLASS_ALG, :prediction_feature => FEATURE_URI, +# :algorithm_params => "feature_generation_uri="+WS_FEATURE_ALG, :split_ratio=>0.8, :random_seed=>5} +# verify_validation +# ensure +# delete_resources +# end # end + def test_nothing + + #puts "testing nothing" + + get '/examples' + + #get '/validation/350',nil,'HTTP_ACCEPT' => "application/rdf+xml" + #get '/validation/350',nil,'HTTP_ACCEPT' => "text/x-yaml" + + #get '/crossvalidation/1',nil,'HTTP_ACCEPT' => "application/rdf+xml" + #get '/crossvalidation/1',nil,'HTTP_ACCEPT' => "text/x-yaml" + + puts last_response.body + end + private def verify_validation (delete=true) |