From ddc100289dd5ff412245bf7ee5d13a6fdb38853f Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 23 Nov 2011 15:57:10 +0100 Subject: add validation to pure-yaml code for serialization --- validation/validation_application.rb | 52 +++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 4 deletions(-) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 7e0e10f..a80396c 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -153,9 +153,13 @@ get '/crossvalidation/:id' do "A crossvalidation resource." content_type "text/html" OpenTox.text_to_html crossvalidation.to_yaml,@subjectid,related_links,description + when "application/serialize" + content_type "application/serialize" + crossvalidation.get_content_as_hash # to load all the stuff + crossvalidation.to_yaml when /application\/x-yaml|\*\/\*/ content_type "application/x-yaml" - crossvalidation.to_yaml + crossvalidation.to_rdf_yaml else raise OpenTox::BadRequestError.new "MIME type '"+request.env['HTTP_ACCEPT'].to_s+"' not supported, valid Accept-Headers: \"application/rdf+xml\", \"application/x-yaml\", \"text/html\"." end @@ -172,13 +176,17 @@ get '/crossvalidation/:id/statistics' do description = "The averaged statistics for the crossvalidation." content_type "text/html" - OpenTox.text_to_html v.to_yaml,@subjectid,related_links,description + OpenTox.text_to_html v.to_rdf_yaml,@subjectid,related_links,description when "application/rdf+xml" content_type "application/rdf+xml" v.to_rdf + when "application/serialize" + content_type "application/serialize" + v.get_content_as_hash # to load all the stuff + v.to_yaml else content_type "application/x-yaml" - v.to_yaml + v.to_rdf_yaml end end @@ -562,6 +570,38 @@ post '/validate_datasets' do return_task(task) end +get '/:id/verify_r_square' do + + #PENDING: this is debug code, move to test-suite + + validation = Validation::Validation.get(params[:id]) + p = validation.compute_validation_stats_with_model(nil, true) + + puts "actual "+p.actual_values.inspect + puts "predicted "+p.predicted_values.inspect + puts "" + + puts "ot r-square "+p.r_square.to_s + puts "ot sample_correlation_coefficient "+p.sample_correlation_coefficient.to_s + puts "ot sample_correlation_coefficient**2 "+(p.sample_correlation_coefficient**2).to_s + puts "" + + @@r = RinRuby.new(true,false) unless defined?(@@r) and @@r + @@r.assign "v1",p.actual_values + @@r.assign "v2",p.predicted_values + puts "r cor "+@@r.pull("cor(v1,v2)").to_s + # @@r.eval "ttest = t.test(v1,v2,paired=T)" + # t = @@r.pull "ttest$statistic" + @@r.eval "fit <- lm(v1 ~ v2)" + @@r.eval "sum <- summary(fit)" + puts "r r-square "+@@r.pull("sum$r.squared").to_s + puts "r adjusted-r-square "+@@r.pull("sum$adj.r.squared").to_s + + @@r.quit + @@r = nil + +end + get '/:id/predictions' do LOGGER.info "get validation predictions "+params.inspect begin @@ -627,9 +667,13 @@ get '/:id' do "All validations: "+url_for("/",:full)+"\n"+ "All validation reports: "+url_for("/report/validation",:full) OpenTox.text_to_html validation.to_yaml,@subjectid,related_links,description + when "application/serialize" + content_type "application/serialize" + validation.get_content_as_hash # to load all the stuff + validation.to_yaml else #default is yaml content_type "application/x-yaml" - validation.to_yaml + validation.to_rdf_yaml end end -- cgit v1.2.3 From d5a53dec3b7e97bb8749dc4cd9afefc08ee15768 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 23 Nov 2011 16:02:12 +0100 Subject: adjust yaml version for to-html --- validation/validation_application.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index a80396c..d3c4c2f 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -152,7 +152,7 @@ get '/crossvalidation/:id' do description = "A crossvalidation resource." content_type "text/html" - OpenTox.text_to_html crossvalidation.to_yaml,@subjectid,related_links,description + OpenTox.text_to_html crossvalidation.to_rdf_yaml,@subjectid,related_links,description when "application/serialize" content_type "application/serialize" crossvalidation.get_content_as_hash # to load all the stuff @@ -666,7 +666,7 @@ get '/:id' do "Get validation predictions: "+url_for("/"+params[:id]+"/predictions",:full)+"\n"+ "All validations: "+url_for("/",:full)+"\n"+ "All validation reports: "+url_for("/report/validation",:full) - OpenTox.text_to_html validation.to_yaml,@subjectid,related_links,description + OpenTox.text_to_html validation.to_rdf_yaml,@subjectid,related_links,description when "application/serialize" content_type "application/serialize" validation.get_content_as_hash # to load all the stuff -- cgit v1.2.3 From c16535924b2941e9e7316d982d8836cdb83e6b5a Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 23 Nov 2011 16:23:40 +0100 Subject: fix load validations for serialization --- validation/validation_application.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index d3c4c2f..4b6763a 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -155,7 +155,7 @@ get '/crossvalidation/:id' do OpenTox.text_to_html crossvalidation.to_rdf_yaml,@subjectid,related_links,description when "application/serialize" content_type "application/serialize" - crossvalidation.get_content_as_hash # to load all the stuff + crossvalidation.inspect # to load all the stuff crossvalidation.to_yaml when /application\/x-yaml|\*\/\*/ content_type "application/x-yaml" @@ -182,7 +182,7 @@ get '/crossvalidation/:id/statistics' do v.to_rdf when "application/serialize" content_type "application/serialize" - v.get_content_as_hash # to load all the stuff + v.inspect # to load all the stuff v.to_yaml else content_type "application/x-yaml" @@ -669,7 +669,7 @@ get '/:id' do OpenTox.text_to_html validation.to_rdf_yaml,@subjectid,related_links,description when "application/serialize" content_type "application/serialize" - validation.get_content_as_hash # to load all the stuff + validation.inspect # to load all the stuff validation.to_yaml else #default is yaml content_type "application/x-yaml" -- cgit v1.2.3 From cf60c03db2481d3816e63f058a7ed12d905ac833 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Fri, 25 Nov 2011 09:07:50 +0100 Subject: add r-square plot, fix prediction updating, add weighted sample-correlation-coefficient --- validation/validation_application.rb | 32 -------------------------------- 1 file changed, 32 deletions(-) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 4b6763a..4b2a2d9 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -570,38 +570,6 @@ post '/validate_datasets' do return_task(task) end -get '/:id/verify_r_square' do - - #PENDING: this is debug code, move to test-suite - - validation = Validation::Validation.get(params[:id]) - p = validation.compute_validation_stats_with_model(nil, true) - - puts "actual "+p.actual_values.inspect - puts "predicted "+p.predicted_values.inspect - puts "" - - puts "ot r-square "+p.r_square.to_s - puts "ot sample_correlation_coefficient "+p.sample_correlation_coefficient.to_s - puts "ot sample_correlation_coefficient**2 "+(p.sample_correlation_coefficient**2).to_s - puts "" - - @@r = RinRuby.new(true,false) unless defined?(@@r) and @@r - @@r.assign "v1",p.actual_values - @@r.assign "v2",p.predicted_values - puts "r cor "+@@r.pull("cor(v1,v2)").to_s - # @@r.eval "ttest = t.test(v1,v2,paired=T)" - # t = @@r.pull "ttest$statistic" - @@r.eval "fit <- lm(v1 ~ v2)" - @@r.eval "sum <- summary(fit)" - puts "r r-square "+@@r.pull("sum$r.squared").to_s - puts "r adjusted-r-square "+@@r.pull("sum$adj.r.squared").to_s - - @@r.quit - @@r = nil - -end - get '/:id/predictions' do LOGGER.info "get validation predictions "+params.inspect begin -- cgit v1.2.3 From 7565aeb930c9b24a677b65b89d62cc2db6318cee Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 7 Dec 2011 16:26:41 +0100 Subject: add computation of classification prediciton probabilities --- validation/validation_application.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 4b2a2d9..0647b10 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -190,6 +190,15 @@ get '/crossvalidation/:id/statistics' do end end +get '/crossvalidation/:id/statistics/probabilities' do + + LOGGER.info "get crossvalidation statistics for crossvalidation with id "+params[:id].to_s + v = Validation::Validation.from_cv_statistics( params[:id], @subjectid ) + props = v.probabilities(params[:confidence].to_s.to_f,params[:prediction].to_s) + content_type "text/x-yaml" + props.to_yaml +end + delete '/crossvalidation/:id/?' do LOGGER.info "delete crossvalidation with id "+params[:id].to_s content_type "text/plain" @@ -570,6 +579,22 @@ post '/validate_datasets' do return_task(task) end +get '/:id/probabilities' do + LOGGER.info "get validation probabilities "+params.inspect + + begin + validation = Validation::Validation.get(params[:id]) + rescue ActiveRecord::RecordNotFound => ex + raise OpenTox::NotFoundError.new "Validation '#{params[:id]}' not found." + end + validation.subjectid = @subjectid + raise OpenTox::BadRequestError.new "Validation '"+params[:id].to_s+"' not finished" unless validation.finished + props = validation.probabilities(params[:confidence].to_s.to_f,params[:prediction].to_s) + content_type "text/x-yaml" + props.to_yaml +end + + get '/:id/predictions' do LOGGER.info "get validation predictions "+params.inspect begin -- cgit v1.2.3 From d02b54b2c58d2d71e29700bbedbb38768d6c9e35 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Tue, 13 Dec 2011 11:20:04 +0100 Subject: add filtering of validation reports --- validation/validation_application.rb | 104 +++++++++++++++++------------------ 1 file changed, 52 insertions(+), 52 deletions(-) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 0647b10..f126679 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -226,34 +226,34 @@ end # Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ).collect{ |v| v.validation_uri.to_s }.join("\n")+"\n" #end -get '/crossvalidation/:id/predictions' do - LOGGER.info "get predictions for crossvalidation with id "+params[:id].to_s - begin - #crossvalidation = Validation::Crossvalidation.find(params[:id]) - crossvalidation = Validation::Crossvalidation.get(params[:id]) - rescue ActiveRecord::RecordNotFound => ex - raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found." - end - raise OpenTox::BadRequestError.new "Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished - - content_type "application/x-yaml" - validations = Validation::Validation.find( :crossvalidation_id => params[:id], :validation_type => "crossvalidation" ) - p = Lib::OTPredictions.to_array( validations.collect{ |v| v.compute_validation_stats_with_model(nil, true) } ).to_yaml - - case request.env['HTTP_ACCEPT'].to_s - when /text\/html/ - content_type "text/html" - description = - "The crossvalidation predictions as (yaml-)array." - related_links = - "All crossvalidations: "+url_for("/crossvalidation",:full)+"\n"+ - "Correspoding crossvalidation: "+url_for("/crossvalidation/"+params[:id],:full) - OpenTox.text_to_html p,@subjectid, related_links, description - else - content_type "text/x-yaml" - p - end -end +#get '/crossvalidation/:id/predictions' do +# LOGGER.info "get predictions for crossvalidation with id "+params[:id].to_s +# begin +# #crossvalidation = Validation::Crossvalidation.find(params[:id]) +# crossvalidation = Validation::Crossvalidation.get(params[:id]) +# rescue ActiveRecord::RecordNotFound => ex +# raise OpenTox::NotFoundError.new "Crossvalidation '#{params[:id]}' not found." +# end +# raise OpenTox::BadRequestError.new "Crossvalidation '"+params[:id].to_s+"' not finished" unless crossvalidation.finished +# +# content_type "application/x-yaml" +# validations = Validation::Validation.find( :crossvalidation_id => params[:id], :validation_type => "crossvalidation" ) +# p = Lib::OTPredictions.to_array( validations.collect{ |v| v.compute_validation_stats_with_model(nil, true) } ).to_yaml +# +# case request.env['HTTP_ACCEPT'].to_s +# when /text\/html/ +# content_type "text/html" +# description = +# "The crossvalidation predictions as (yaml-)array." +# related_links = +# "All crossvalidations: "+url_for("/crossvalidation",:full)+"\n"+ +# "Correspoding crossvalidation: "+url_for("/crossvalidation/"+params[:id],:full) +# OpenTox.text_to_html p,@subjectid, related_links, description +# else +# content_type "text/x-yaml" +# p +# end +#end get '/?' do @@ -595,30 +595,30 @@ get '/:id/probabilities' do end -get '/:id/predictions' do - LOGGER.info "get validation predictions "+params.inspect - begin - #validation = Validation::Validation.find(params[:id]) - validation = Validation::Validation.get(params[:id]) - rescue ActiveRecord::RecordNotFound => ex - raise OpenTox::NotFoundError.new "Validation '#{params[:id]}' not found." - end - raise OpenTox::BadRequestError.new "Validation '"+params[:id].to_s+"' not finished" unless validation.finished - p = validation.compute_validation_stats_with_model(nil, true) - case request.env['HTTP_ACCEPT'].to_s - when /text\/html/ - content_type "text/html" - description = - "The validation predictions as (yaml-)array." - related_links = - "All validations: "+url_for("/",:full)+"\n"+ - "Correspoding validation: "+url_for("/"+params[:id],:full) - OpenTox.text_to_html p.to_array.to_yaml,@subjectid, related_links, description - else - content_type "text/x-yaml" - p.to_array.to_yaml - end -end +#get '/:id/predictions' do +# LOGGER.info "get validation predictions "+params.inspect +# begin +# #validation = Validation::Validation.find(params[:id]) +# validation = Validation::Validation.get(params[:id]) +# rescue ActiveRecord::RecordNotFound => ex +# raise OpenTox::NotFoundError.new "Validation '#{params[:id]}' not found." +# end +# raise OpenTox::BadRequestError.new "Validation '"+params[:id].to_s+"' not finished" unless validation.finished +# p = validation.compute_validation_stats_with_model(nil, true) +# case request.env['HTTP_ACCEPT'].to_s +# when /text\/html/ +# content_type "text/html" +# description = +# "The validation predictions as (yaml-)array." +# related_links = +# "All validations: "+url_for("/",:full)+"\n"+ +# "Correspoding validation: "+url_for("/"+params[:id],:full) +# OpenTox.text_to_html p.to_array.to_yaml,@subjectid, related_links, description +# else +# content_type "text/x-yaml" +# p.to_array.to_yaml +# end +#end #get '/:id/:attribute' do # LOGGER.info "access validation attribute "+params.inspect -- cgit v1.2.3 From 4a4b743704b819dec39c672d9550038f45484160 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 4 Jan 2012 12:23:33 +0100 Subject: fix validation aa errors, add more verbose error msg for propbabilites feature --- validation/validation_application.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index f126679..c02b5f3 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -193,6 +193,7 @@ end get '/crossvalidation/:id/statistics/probabilities' do LOGGER.info "get crossvalidation statistics for crossvalidation with id "+params[:id].to_s + raise OpenTox::BadRequestError.new("Missing params, plz give confidence and prediction") unless params[:confidence] and params[:prediction] v = Validation::Validation.from_cv_statistics( params[:id], @subjectid ) props = v.probabilities(params[:confidence].to_s.to_f,params[:prediction].to_s) content_type "text/x-yaml" @@ -585,10 +586,11 @@ get '/:id/probabilities' do begin validation = Validation::Validation.get(params[:id]) rescue ActiveRecord::RecordNotFound => ex - raise OpenTox::NotFoundError.new "Validation '#{params[:id]}' not found." + raise OpenTox::NotFoundError.new("Validation '#{params[:id]}' not found.") end validation.subjectid = @subjectid - raise OpenTox::BadRequestError.new "Validation '"+params[:id].to_s+"' not finished" unless validation.finished + raise OpenTox::BadRequestError.new("Validation '"+params[:id].to_s+"' not finished") unless validation.finished + raise OpenTox::BadRequestError.new("Missing params, plz give confidence and prediction") unless params[:confidence] and params[:prediction] props = validation.probabilities(params[:confidence].to_s.to_f,params[:prediction].to_s) content_type "text/x-yaml" props.to_yaml -- cgit v1.2.3 From 9f2be4ca3bded1543142f5e3654693ce65aadb44 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 18 Jan 2012 17:54:11 +0100 Subject: add super-stratification split for training test splitting --- validation/validation_application.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index c02b5f3..cda09fa 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -453,8 +453,9 @@ post '/training_test_split' do raise OpenTox::BadRequestError.new "algorithm_uri missing" unless params[:algorithm_uri].to_s.size>0 raise OpenTox::BadRequestError.new "prediction_feature missing" unless params[:prediction_feature].to_s.size>0 task = OpenTox::Task.create( "Perform training test split validation", url_for("/training_test_split", :full) ) do |task| #, params + strat = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] params.merge!( Validation::Util.train_test_dataset_split(params[:dataset_uri], params[:prediction_feature], - @subjectid, params[:split_ratio], params[:random_seed], OpenTox::SubTask.create(task,0,33))) + @subjectid, strat, params[:split_ratio], params[:random_seed], OpenTox::SubTask.create(task,0,33))) v = Validation::Validation.create :validation_type => "training_test_split", :training_dataset_uri => params[:training_dataset_uri], :test_dataset_uri => params[:test_dataset_uri], @@ -544,8 +545,9 @@ end post '/plain_training_test_split' do LOGGER.info "creating pure training test split "+params.inspect raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] - - result = Validation::Util.train_test_dataset_split(params[:dataset_uri], params[:prediction_feature], params[:split_ratio], params[:random_seed]) + strat = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] + result = Validation::Util.train_test_dataset_split(params[:dataset_uri], params[:prediction_feature], @subjectid, + strat, params[:split_ratio], params[:random_seed]) content_type "text/uri-list" result[:training_dataset_uri]+"\n"+result[:test_dataset_uri]+"\n" end -- cgit v1.2.3 From eda4b9687ca4b093b7f194b6d0b2e58ce7eed3b2 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Wed, 18 Jan 2012 18:56:48 +0100 Subject: wrap plain split into taks --- validation/validation_application.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index cda09fa..279cd14 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -543,13 +543,16 @@ post '/cleanup_datasets/?' do end post '/plain_training_test_split' do - LOGGER.info "creating pure training test split "+params.inspect - raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] + LOGGER.info "creating pure training test split "+params.inspect + raise OpenTox::BadRequestError.new "dataset_uri missing" unless params[:dataset_uri] + task = OpenTox::Task.create( "Create data-split", url_for("/plain_training_test_split", :full) ) do |task| strat = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] result = Validation::Util.train_test_dataset_split(params[:dataset_uri], params[:prediction_feature], @subjectid, strat, params[:split_ratio], params[:random_seed]) content_type "text/uri-list" result[:training_dataset_uri]+"\n"+result[:test_dataset_uri]+"\n" + end + return_task(task) end post '/validate_datasets' do -- cgit v1.2.3 From f2933ba1b51ba5dc28bc649d0919d6d7dce14721 Mon Sep 17 00:00:00 2001 From: rautenberg Date: Tue, 6 Mar 2012 16:10:20 +0100 Subject: subjectid from helper --- validation/validation_application.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'validation/validation_application.rb') diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 279cd14..32ca971 100755 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -51,7 +51,7 @@ post '/crossvalidation/?' do cv_params = { :dataset_uri => params[:dataset_uri], :algorithm_uri => params[:algorithm_uri], :loo => "false", - :subjectid => params[:subjectid] } + :subjectid => @subjectid } [ :num_folds, :random_seed ].each{ |sym| cv_params[sym] = params[sym] if params[sym] } cv_params[:stratified] = (params[:stratified].size>0 && params[:stratified]!="false" && params[:stratified]!="0") if params[:stratified] cv = Validation::Crossvalidation.create cv_params -- cgit v1.2.3