diff options
-rw-r--r-- | EXAMPLES | 6 | ||||
-rw-r--r-- | db/migrate/000_drop_validations.rb | 8 | ||||
-rw-r--r-- | db/migrate/001_init_validation.rb | 12 | ||||
-rw-r--r-- | db/migrate/002_init_reports.rb | 36 | ||||
-rw-r--r-- | example.rb | 5 | ||||
-rw-r--r-- | lib/rdf_provider.rb | 54 | ||||
-rw-r--r-- | lib/validation_db.rb | 33 | ||||
-rw-r--r-- | report/environment.rb | 16 | ||||
-rw-r--r-- | report/report_application.rb | 15 | ||||
-rw-r--r-- | report/report_format.rb | 32 | ||||
-rw-r--r-- | report/report_persistance.rb | 172 | ||||
-rw-r--r-- | report/report_service.rb | 30 | ||||
-rw-r--r-- | report/report_test.rb | 22 | ||||
-rw-r--r-- | report/validation_access.rb | 13 | ||||
-rw-r--r-- | test/test_examples_util.rb | 99 | ||||
-rw-r--r-- | validation/validation_application.rb | 17 | ||||
-rw-r--r-- | validation/validation_format.rb | 115 | ||||
-rw-r--r-- | validation/validation_service.rb | 18 | ||||
-rw-r--r-- | validation/validation_test.rb | 25 |
19 files changed, 479 insertions, 249 deletions
@@ -31,6 +31,8 @@ search available validations with url-encoded parameters >>> curl <validation_service>?training_dataset_uri=<training_dataset_uri>&algorithm_uri=<algorithm_uri> +Hint: you can perform a pattern search (instead of an exact match search) by adding a _like to the parameter, i.e. training_dataset_uri_like + result example (accept-header: application/rdf-xml) <<< not yet supported @@ -200,6 +202,8 @@ get validation report Supported formats (accept-headers): * "text/xml" content of report in docbook-article format * "text/html" report formated with default docbook-article-xsl +* "text/x-yaml" returns report object with meta-info (without the actual report content) +* "application/rdf+xml" returns report object with meta-info (without the actual report content) Hint: Visit <validation_service>/report/validation/<validation_report_id> with a browser to see the report in html format @@ -239,6 +243,8 @@ get crossvalidation report Supported formats (accept-headers): * "text/xml" content of report in docbook-article format * "text/html" report formated with default docbook-article-xsl +* "text/x-yaml" returns report object with meta-info (without the actual report content) +* "application/rdf+xml" returns report object with meta-info (without the actual report content) Hint: Visit <validation_service>/report/crossvalidation/<crossvalidation_report_id> with a browser to see the report in html format diff --git a/db/migrate/000_drop_validations.rb b/db/migrate/000_drop_validations.rb index 4b0288d..fb70967 100644 --- a/db/migrate/000_drop_validations.rb +++ b/db/migrate/000_drop_validations.rb @@ -1,13 +1,13 @@ class DropValidations < ActiveRecord::Migration def self.up - drop_table :validations if table_exists? :validations - drop_table :crossvalidations if table_exists? :crossvalidations + # drop_table :validations if table_exists? :validations + # drop_table :crossvalidations if table_exists? :crossvalidations end def self.down - drop_table :validations if table_exists? :validations - drop_table :crossvalidations if table_exists? :crossvalidations + #drop_table :validations if table_exists? :validations + #drop_table :crossvalidations if table_exists? :crossvalidations end end diff --git a/db/migrate/001_init_validation.rb b/db/migrate/001_init_validation.rb index d38afd7..93d8d2f 100644 --- a/db/migrate/001_init_validation.rb +++ b/db/migrate/001_init_validation.rb @@ -4,7 +4,7 @@ class InitValidation < ActiveRecord::Migration create_table :crossvalidations do |t| - [:crossvalidation_uri, #accesss to :uri somehow does not work, create uri-function in object + [:crossvalidation_uri, :algorithm_uri, :dataset_uri ].each do |p| t.column p, :string, :limit => 255 @@ -27,14 +27,15 @@ class InitValidation < ActiveRecord::Migration create_table :validations do |t| - [:validation_uri, #accesss to :uri somehow does not work, create uri-function in obejct + [:validation_uri, :model_uri, :algorithm_uri, :training_dataset_uri, :test_target_dataset_uri, :test_dataset_uri, :prediction_dataset_uri, - :prediction_feature ].each do |p| + :prediction_feature, + :crossvalidation_uri].each do |p| t.column p, :string, :limit => 255 end @@ -57,8 +58,7 @@ class InitValidation < ActiveRecord::Migration end def self.down - drop_table :validations - drop_table :crossvalidations + drop_table :validations if table_exists? :validations + drop_table :crossvalidations if table_exists? :crossvalidations end end - diff --git a/db/migrate/002_init_reports.rb b/db/migrate/002_init_reports.rb new file mode 100644 index 0000000..8029223 --- /dev/null +++ b/db/migrate/002_init_reports.rb @@ -0,0 +1,36 @@ + +class InitReports < ActiveRecord::Migration + def self.up + + create_table :report_datum do |t| + + [:report_uri, + :report_type + ].each do |p| + t.column p, :string, :limit => 255 + end + + [:created_at ].each do |p| + t.column p, :datetime + end + + [:validation_uris, :crossvalidation_uris, :model_uris, :algorithm_uris].each do |p| + t.column(p, :text, :limit => 16320) + end + end + end + + def self.down + drop_table :report_datum if table_exists? :report_datum + if @@config[:reports] and @@config[:reports][:report_dir] + ["validation", "crossvalidation", "algorithm_comparison"].each do |t| + dir = File.join(@@config[:reports][:report_dir],t) + if File.exist?(dir) + puts "deleting dir "+dir.to_s + FileUtils.rm_rf(dir) + end + end + end + end +end + @@ -56,6 +56,7 @@ class Example ActiveRecord::Base.logger = Logger.new("/dev/null") ActiveRecord::Migrator.migrate('db/migrate', 0 ) ActiveRecord::Migrator.migrate('db/migrate', 1 ) + ActiveRecord::Migrator.migrate('db/migrate', 2 ) #delete_all(@@config[:services]["opentox-dataset"]) log OpenTox::RestClientWrapper.delete @@config[:services]["opentox-dataset"] @@ -85,11 +86,11 @@ class Example log "create validation report" rep = Reports::ReportService.new(File.join(@@config[:services]["opentox-validation"],"report")) rep.delete_all_reports("validation") - rep.create_report("validation",v.uri) + rep.create_report("validation",v.validation_uri) log "create crossvalidation report" rep.delete_all_reports("crossvalidation") - rep.create_report("crossvalidation",cv.uri) + rep.create_report("crossvalidation",cv.crossvalidation_uri) log "done" @@summary diff --git a/lib/rdf_provider.rb b/lib/rdf_provider.rb index e9d0f2f..5894cb1 100644 --- a/lib/rdf_provider.rb +++ b/lib/rdf_provider.rb @@ -1,4 +1,13 @@ +class String + def convert_underscore + gsub(/_./) do |m| + m.gsub!(/^_/,"") + m.upcase + end + end +end + module Lib module RDFProvider @@ -25,29 +34,42 @@ module Lib raise "not implemented" end - def literal?( property ) - raise "not yet implemented" + def to_yaml + get_content_as_hash.to_yaml end - def literal_name( property ) - raise "not yet implemented" + def rdf_ignore?( prop ) + self.class::IGNORE.index( prop ) != nil end - def object_property?( property ) - raise "not yet implemented" + def literal?( prop ) + self.class::LITERALS.index( prop ) != nil end - def object_property_name( property ) - raise "not yet implemented" + def literal_name( prop ) + if self.class::LITERAL_NAMES.has_key?(prop) + self.class::LITERAL_NAMES[prop] + else + OT[prop.to_s.convert_underscore] + end end - - def class?( property ) - raise "not yet implemented" + + def object_property?( prop ) + self.class::OBJECT_PROPERTIES.has_key?( prop ) + end + + def object_property_name( prop ) + return self.class::OBJECT_PROPERTIES[ prop ] end - def class_name( property ) - raise "not yet implemented" + def class?(prop) + self.class::CLASSES.has_key?( prop ) end + + def class_name( prop ) + return self.class::CLASSES[ prop ] + end + end class HashToOwl @@ -82,7 +104,9 @@ module Lib LOGGER.warn "skipping nil value: "+k.to_s next end - if v.is_a?(Hash) + if @rdf_provider.rdf_ignore?(k) + #do nothing + elsif v.is_a?(Hash) new_node = add_class( k, node ) recursiv_add_content( v, new_node ) elsif v.is_a?(Array) @@ -98,8 +122,6 @@ module Lib set_literal( k, v, node) elsif @rdf_provider.object_property?(k) add_object_property( k, v, node) - elsif [ :uri, :id ].index(k)!=nil - #skip else raise "illegal value k:"+k.to_s+" v:"+v.to_s end diff --git a/lib/validation_db.rb b/lib/validation_db.rb index a3c9593..b7cafd8 100644 --- a/lib/validation_db.rb +++ b/lib/validation_db.rb @@ -2,28 +2,31 @@ [ 'rubygems', 'datamapper' ].each do |lib| require lib end - require "lib/merge.rb" require 'active_record' -ActiveRecord::Base.establish_connection( - :adapter => @@config[:database][:adapter], - :host => @@config[:database][:host], - :database => @@config[:database][:database], - :username => @@config[:database][:username], - :password => @@config[:database][:password] -) +require 'ar-extensions' +unless ActiveRecord::Base.connected? + ActiveRecord::Base.establish_connection( + :adapter => @@config[:database][:adapter], + :host => @@config[:database][:host], + :database => @@config[:database][:database], + :username => @@config[:database][:username], + :password => @@config[:database][:password] + ) + ActiveRecord::Base.logger = Logger.new("/dev/null") +end module Lib - VAL_PROPS_GENERAL = [ :id, :uri, :model_uri, :algorithm_uri, :training_dataset_uri, :prediction_feature, + VAL_PROPS_GENERAL = [ :validation_uri, :model_uri, :algorithm_uri, :training_dataset_uri, :prediction_feature, :test_dataset_uri, :test_target_dataset_uri, :prediction_dataset_uri, :created_at ] VAL_PROPS_SUM = [ :num_instances, :num_without_class, :num_unpredicted ] VAL_PROPS_AVG = [:real_runtime, :percent_without_class, :percent_unpredicted ] VAL_PROPS = VAL_PROPS_GENERAL + VAL_PROPS_SUM + VAL_PROPS_AVG # :crossvalidation_info - VAL_CV_PROPS = [ :crossvalidation_id, :crossvalidation_fold ] + VAL_CV_PROPS = [ :crossvalidation_id, :crossvalidation_uri, :crossvalidation_fold ] # :classification_statistics VAL_CLASS_PROPS_SINGLE_SUM = [ :num_correct, :num_incorrect, :confusion_matrix ] @@ -48,7 +51,7 @@ module Lib VAL_REGR_PROPS = [ :root_mean_squared_error, :mean_absolute_error, :r_square, :target_variance_actual, :target_variance_predicted ] CROSS_VAL_PROPS = [:dataset_uri, :num_folds, :stratified, :random_seed] - CROSS_VAL_PROPS_REDUNDANT = [:algorithm_uri, :created_at] + CROSS_VAL_PROPS + CROSS_VAL_PROPS_REDUNDANT = [:crossvalidation_uri, :algorithm_uri, :created_at] + CROSS_VAL_PROPS ALL_PROPS = VAL_PROPS + VAL_CV_PROPS + VAL_CLASS_PROPS_EXTENDED + VAL_REGR_PROPS + CROSS_VAL_PROPS @@ -57,14 +60,10 @@ module Lib VAL_MERGE_AVG = VAL_PROPS_AVG + VAL_CLASS_PROPS_SINGLE_AVG + VAL_CLASS_PROPS_PER_CLASS_AVG + VAL_REGR_PROPS class Validation < ActiveRecord::Base - def uri - self.validation_uri - end + serialize :classification_statistics + serialize :regression_statistics end class Crossvalidation < ActiveRecord::Base - def uri - self.crossvalidation_uri - end end end diff --git a/report/environment.rb b/report/environment.rb index c64d40d..154374f 100644 --- a/report/environment.rb +++ b/report/environment.rb @@ -10,12 +10,26 @@ require 'opentox-ruby-api-wrapper' require 'fileutils' require 'mime/types' require 'ruby-plot' - gem 'ruby-plot', '= 0.0.2' +require 'active_record' +require 'ar-extensions' +unless ActiveRecord::Base.connected? + ActiveRecord::Base.establish_connection( + :adapter => @@config[:database][:adapter], + :host => @@config[:database][:host], + :database => @@config[:database][:database], + :username => @@config[:database][:username], + :password => @@config[:database][:password] + ) + ActiveRecord::Base.logger = Logger.new("/dev/null") +end + module Reports end +require "lib/rdf_provider.rb" + require "report/plot_factory.rb" require "report/xml_report.rb" require "report/xml_report_util.rb" diff --git a/report/report_application.rb b/report/report_application.rb index 29367be..dc61e3a 100644 --- a/report/report_application.rb +++ b/report/report_application.rb @@ -32,10 +32,10 @@ get '/report/?' do end end -get '/report/:type' do +get '/report/:report_type' do perform do |rs| content_type "text/uri-list" - rs.get_all_reports(params[:type]) + rs.get_all_reports(params[:report_type], params) end end @@ -60,8 +60,15 @@ get '/report/:type/:id' do end #request.env['HTTP_ACCEPT'] = "application/pdf" - content_type Reports::ReportFormat.get_format(accept_header) - result = body(File.new( rs.get_report(params[:type],params[:id],accept_header) )) + report = rs.get_report(params[:type],params[:id],accept_header) + format = Reports::ReportFormat.get_format(accept_header) + content_type format + #PENDING: get_report should return file or string, check for result.is_file instead of format + if format=="text/x-yaml" or format=="application/rdf+xml" + report + else + result = body(File.new(report)) + end end end diff --git a/report/report_format.rb b/report/report_format.rb index e9a5645..2eeb546 100644 --- a/report/report_format.rb +++ b/report/report_format.rb @@ -10,22 +10,30 @@ ENV['SAXON_JAR'] = "saxonhe9-2-0-3j/saxon9he.jar" unless ENV['SAXON_JAR'] # module Reports::ReportFormat - RF_XML = "xml" - RF_HTML = "html" - RF_PDF = "pdf" - REPORT_FORMATS = [RF_XML, RF_HTML, RF_PDF] - CONTENT_TYPES = {"text/xml"=>RF_XML,"text/html"=>RF_HTML,"application/pdf"=>RF_PDF} + CONTENT_TYPES = ["text/x-yaml","text/html","application/rdf+xml", "text/xml","application/pdf"] # returns report-format, according to header value def self.get_format(accept_header_value) - begin - content_type = MIMEParse::best_match(CONTENT_TYPES.keys, accept_header_value) + content_type = MIMEParse::best_match(CONTENT_TYPES, accept_header_value) raise RuntimeException.new unless content_type rescue - raise Reports::BadRequest.new("Accept header '"+accept_header_value.to_s+"' not supported, supported types are "+CONTENT_TYPES.keys.join(", ")) + raise Reports::BadRequest.new("Accept header '"+accept_header_value.to_s+"' not supported, supported types are "+CONTENT_TYPES.join(", ")) + end + return content_type + end + + def self.get_filename_extension(format) + case format + when "text/xml" + "xml" + when "text/html" + "html" + when "application/pdf" + "pdf" + else + raise "invalid format type for file extensions: "+format.to_s end - return CONTENT_TYPES[content_type] end # formats a report from xml into __format__ @@ -33,7 +41,7 @@ module Reports::ReportFormat # * the new format can be found in __dest_filame__ def self.format_report(directory, xml_filename, dest_filename, format, overwrite=false, params={}) - raise "cannot format to XML" if format==RF_XML + raise "cannot format to XML" if format=="text/xml" raise "directory does not exist: "+directory.to_s unless File.directory?directory.to_s xml_file = directory.to_s+"/"+xml_filename.to_s raise "xml file not found: "+xml_file unless File.exist?xml_file @@ -41,9 +49,9 @@ module Reports::ReportFormat raise "destination file already exists: "+dest_file if (File.exist?(dest_file) && !overwrite) case format - when RF_HTML + when "text/html" format_report_to_html(directory, xml_filename, dest_filename, params[:css_style_sheet]) - when RF_PDF + when "application/pdf" raise "pdf conversion not supported yet" else raise "unknown format type" diff --git a/report/report_persistance.rb b/report/report_persistance.rb index b6c4077..5df0e20 100644 --- a/report/report_persistance.rb +++ b/report/report_persistance.rb @@ -1,7 +1,4 @@ -ENV['REPORT_DIR'] = File.join(FileUtils.pwd,"reports") unless ENV['REPORT_DIR'] - - # = Reports::ReportPersistance # # service that stores reports (Reports::ReportConent), and provides access in various formats @@ -13,7 +10,7 @@ class Reports::ReportPersistance # call-seq: # list_reports(type) => Array # - def list_reports(type) + def list_reports(type, filter_params) raise "not implemented" end @@ -70,55 +67,26 @@ end class Reports::FileReportPersistance < Reports::ReportPersistance def initialize() - @report_dir = ENV['REPORT_DIR'] + raise "pls specify report-directory (:reports -> :report_dir) in config file" unless @@config[:reports] and @@config[:reports][:report_dir] + @report_dir = @@config[:reports][:report_dir] FileUtils.mkdir @report_dir.to_s unless File.directory?(@report_dir) raise "report cannot be found nor created" unless File.directory?(@report_dir) LOGGER.debug "reports are stored in "+@report_dir.to_s end - def list_reports(type) + def list_reports(type, filter_params=nil) + raise "filter params not supported" if filter_params (Dir.new(type_directory(type)).entries - [".", ".."]).sort{|x,y| x.to_i <=> y.to_i} end - def new_report(report_content, type) - - LOGGER.debug "storing new report of type "+type.to_s - - type_dir = type_directory(type) - raise "type dir '"+type_dir+"' cannot be found nor created" unless File.directory?(type_dir) - - id = 1 - while File.exist?( type_dir+"/"+id.to_s ) - id += 1 - end - report_dir = type_dir+"/"+id.to_s - FileUtils.mkdir(report_dir) - raise "report dir '"+report_dir+"' cannot be created" unless File.directory?(report_dir) - - xml_filename = report_dir+"/report.xml" - xml_file = File.new(xml_filename, "w") - report_content.xml_report.write_to(xml_file, id) - xml_file.close - if (report_content.tmp_files) - report_content.tmp_files.each do |k,v| - tmp_filename = report_dir+"/"+k - raise "tmp-file '"+tmp_filename.to_s+"' already exists" if File.exist?(tmp_filename) - raise "tmp-file '"+v.to_s+"' not found" unless File.exist?(v) - FileUtils.mv(v.to_s,tmp_filename) - raise "could not move tmp-file to '"+tmp_filename.to_s+"'" unless File.exist?(tmp_filename) - end - end - return id - end - def get_report(type, id, format, force_formating, params) report_dir = report_directory(type, id) raise_report_not_found(type, id) unless File.directory?(report_dir) - filename = "report."+format + filename = "report."+Reports::ReportFormat.get_filename_extension(format) file_path = report_dir+"/"+filename - + return file_path if File.exist?(file_path) && !force_formating Reports::ReportFormat.format_report(report_dir, "report.xml", filename, format, force_formating, params) @@ -151,6 +119,46 @@ class Reports::FileReportPersistance < Reports::ReportPersistance raise "not valid report id format" unless id.to_s =~ /[0-9]+/ end + def new_report(report_content, type, meta_data=nil, uri_provider=nil) + new_report_with_id(report_content, type) + end + + protected + def new_report_with_id(report_content, type, force_id=nil) + LOGGER.debug "storing new report of type "+type.to_s + + type_dir = type_directory(type) + raise "type dir '"+type_dir+"' cannot be found nor created" unless File.directory?(type_dir) + + if (force_id==nil) + id = 1 + while File.exist?( type_dir+"/"+id.to_s ) + id += 1 + end + else + raise "report with id '"+force_id.to_s+"' already exists, file system not consistent with db" if File.exist?( type_dir+"/"+force_id.to_s ) + id = force_id + end + report_dir = type_dir+"/"+id.to_s + FileUtils.mkdir(report_dir) + raise "report dir '"+report_dir+"' cannot be created" unless File.directory?(report_dir) + + xml_filename = report_dir+"/report.xml" + xml_file = File.new(xml_filename, "w") + report_content.xml_report.write_to(xml_file, id) + xml_file.close + if (report_content.tmp_files) + report_content.tmp_files.each do |k,v| + tmp_filename = report_dir+"/"+k + raise "tmp-file '"+tmp_filename.to_s+"' already exists" if File.exist?(tmp_filename) + raise "tmp-file '"+v.to_s+"' not found" unless File.exist?(v) + FileUtils.mv(v.to_s,tmp_filename) + raise "could not move tmp-file to '"+tmp_filename.to_s+"'" unless File.exist?(tmp_filename) + end + end + return id + end + private def raise_report_not_found(type, id) raise Reports::NotFound.new("report not found, type:'"+type.to_s+"', id:'"+id.to_s+"'") @@ -169,3 +177,89 @@ class Reports::FileReportPersistance < Reports::ReportPersistance end end + +module Reports + + class ReportData < ActiveRecord::Base + include Lib::RDFProvider + + def get_content_as_hash + map = {} + map[:created_at] = created_at + map[:report_uri] = report_uri + map[:report_type] = report_type + map[:validation_uris] = validation_uris + map[:crossvalidation_uris] = crossvalidation_uris + map[:algorithm_uris] = algorithm_uris + map[:model_uris] = model_uris + map + end + + def rdf_title + "ValidationReport" + end + + def uri + report_uri + end + + LITERALS = [ :created_at, :report_type ] + LITERAL_NAMES = {:created_at => OT["date"] } + OBJECT_PROPERTIES = { :crossvalidation_uris => OT['reportCrossvalidation'], :algorithm_uris => OT['reportAlgorithm'], + :validation_uris => OT['reportValidation'], :model_uris => OT['reportModel'] } + CLASSES = {} + IGNORE = [ :id, :report_uri ] + + serialize :validation_uris + serialize :crossvalidation_uris + serialize :algorithm_uris + serialize :model_uris + end + + class ExtendedFileReportPersistance < FileReportPersistance + + def new_report(report_content, type, meta_data, uri_provider) + raise "report meta data missing" unless meta_data + report = ReportData.new(meta_data) + report.save #to set id + report.attributes = { :report_type => type, :report_uri => uri_provider.get_uri(type, report.id) } + report.save + new_report_with_id(report_content, type, report.id) + end + + def list_reports(type, filter_params=nil) + filter_params = {} unless filter_params + filter_params.each{ |k,v| raise Reports::BadRequest.new("no report-attribute: "+k.to_s) unless ReportData.column_names.include?(k.gsub(/_like$/,"")) } + filter_params[:report_type] = type + ReportData.find(:all, :conditions => filter_params).collect{ |r| r.id } + end + + def get_report(type, id, format, force_formating, params) + + begin + report = ReportData.find(:first, :conditions => {:id => id, :report_type => type}) + rescue ActiveRecord::RecordNotFound + raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") + end + + case format + when "application/rdf+xml" + report.to_rdf + when "text/x-yaml" + report.to_yaml + else + super + end + end + + def delete_report(type, id) + begin + report = ReportData.find(:first, :conditions => {:id => id, :report_type => type}) + rescue ActiveRecord::RecordNotFound + raise Reports::NotFound.new("Report with id='"+id.to_s+"' and type='"+type.to_s+"' not found.") + end + ReportData.delete(id) + super + end + end +end diff --git a/report/report_service.rb b/report/report_service.rb index 854f5f1..d6d0e1a 100644 --- a/report/report_service.rb +++ b/report/report_service.rb @@ -9,7 +9,7 @@ module Reports def initialize(home_uri) LOGGER.info "init report service" @home_uri = home_uri - @persistance = Reports::FileReportPersistance.new + @persistance = Reports::ExtendedFileReportPersistance.new end # lists all available report types, returns list of uris @@ -28,11 +28,11 @@ module Reports # call-seq: # get_all_reports(type) => string # - def get_all_reports(type) + def get_all_reports(type, filter_params) LOGGER.info "get all reports of type '"+type.to_s+"'" check_report_type(type) - @persistance.list_reports(type).collect{ |id| get_uri(type,id) }.join("\n") + @persistance.list_reports(type, filter_params).collect{ |id| get_uri(type,id) }.join("\n") end # creates a report of a certain type, __validation_uris__ must contain be a list of validation or cross-validation-uris @@ -58,7 +58,7 @@ module Reports LOGGER.debug "report created" #step 3: persist report if creation not failed - id = @persistance.new_report(report_content, type) + id = @persistance.new_report(report_content, type, create_meta_data(type, validation_set, validation_uris), self) LOGGER.debug "report persisted with id: '"+id.to_s+"'" return get_uri(type, id) @@ -132,6 +132,28 @@ module Reports end protected + def create_meta_data(type, validation_set, validation_uris) + # the validtion_set contains the resolved single validations + # crossvalidation uris are only added if given as validation_uris - param + meta_data = {} + { :validation_uri => "validation_uris", + :model_uri => "model_uris", + :algorithm_uri => "algorithm_uris" }.each do |key,data| + tmp = [] + validation_set.validations.each do |v| + #tmp << v.send(key) if v.public_methods.include?(key.to_s) and v.send(key) and !tmp.include?(v.send(key)) + tmp << v.send(key) if v.send(key) and !tmp.include?(v.send(key)) + end + meta_data[data.to_sym] = tmp + end + cvs = [] + validation_uris.each do |v| + cvs << v if v =~ /crossvalidation/ and !cvs.include?(v) + end + meta_data[:crossvalidation_uris] = cvs + meta_data + end + def check_report_type(type) raise Reports::NotFound.new("report type not found '"+type.to_s+"'") unless Reports::ReportFactory::REPORT_TYPES.index(type) end diff --git a/report/report_test.rb b/report/report_test.rb index 59aba69..8b68650 100644 --- a/report/report_test.rb +++ b/report/report_test.rb @@ -15,12 +15,10 @@ class Reports::ApplicationTest < Test::Unit::TestCase Sinatra::Application end - - def test_nothing -# get "/match" -# puts last_response.body.to_s + #get "/" #,nil,'HTTP_ACCEPT' => "text/x-yaml"#"application/rdf+xml" + #puts last_response.body.to_s #Reports::XMLReport.generate_demo_xml_report.write_to #raise "stop" @@ -38,14 +36,14 @@ class Reports::ApplicationTest < Test::Unit::TestCase #post 'http://ot.validation.de/report/crossvalidation',:validation_uris=>"http://ot.validation.de/crossvalidation/1" #uri = last_response.body.to_s - val_uris = ["http://localhost/validation/64"]#,"http://localhost/validation/65" ] - - post '/report/validation',:validation_uris=>val_uris.join("\n") - uri = wait_for_task(last_response.body.to_s) - puts uri - id = uri.squeeze("/").split("/")[-1] - get '/report/validation/'+id,nil,'HTTP_ACCEPT' => "text/html" - puts uri +# val_uris = ["http://localhost/validation/64"]#,"http://localhost/validation/65" ] +# +# post '/report/validation',:validation_uris=>val_uris.join("\n") +# uri = wait_for_task(last_response.body.to_s) +# puts uri +# id = uri.squeeze("/").split("/")[-1] +# get '/report/validation/'+id,nil,'HTTP_ACCEPT' => "text/html" +# puts uri #rep = Reports::ReportService.new("http://some.location") #rep.create_report("algorithm_comparison", val_uris) diff --git a/report/validation_access.rb b/report/validation_access.rb index f85698f..7d318af 100644 --- a/report/validation_access.rb +++ b/report/validation_access.rb @@ -59,7 +59,7 @@ class Reports::ValidationDB < Reports::ValidationAccess validation_uris.each do |u| if u.to_s =~ /.*\/crossvalidation\/[0-9]+/ cv_id = u.split("/")[-1].to_i - res += Lib::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.uri.to_s} + res += Lib::Validation.find( :all, :conditions => { :crossvalidation_id => cv_id } ).collect{|v| v.validation_uri.to_s} else res += [u.to_s] end @@ -83,12 +83,12 @@ class Reports::ValidationDB < Reports::ValidationAccess raise Reports::BadRequest.new "no validation found with id "+validation_id.to_s unless v #+" and uri "+uri.to_s unless v (Lib::VAL_PROPS + Lib::VAL_CV_PROPS).each do |p| - validation.send("#{p.to_s}=".to_sym, v[p]) + validation.send("#{p.to_s}=".to_sym, v.send(p)) end {:classification_statistics => Lib::VAL_CLASS_PROPS, :regression_statistics => Lib::VAL_REGR_PROPS}.each do |subset_name,subset_props| - subset = YAML.load(v[subset_name].to_s) + subset = v.send(subset_name) subset_props.each{ |prop| validation.send("#{prop.to_s}=".to_sym, subset[prop]) } if subset end end @@ -184,13 +184,12 @@ class Reports::ValidationWebservice < Reports::ValidationAccess def init_cv(validation) - raise "cv-id not set" unless validation.crossvalidation_id + raise "cv-uri not set" unless validation.crossvalidation_uri - cv_uri = validation.uri.split("/")[0..-3].join("/")+"/crossvalidation/"+validation.crossvalidation_id.to_s begin - data = YAML.load(RestClient.get cv_uri) + data = YAML.load(RestClient.get validation.crossvalidation_uri) rescue => ex - raise Reports::BadRequest.new "cannot get crossvalidation at '"+cv_uri.to_s+"', error msg: "+ex.message + raise Reports::BadRequest.new "cannot get crossvalidation at '"+validation.crossvalidation_uri.to_s+"', error msg: "+ex.message end Lib::CROSS_VAL_PROPS.each do |p| diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb index 7fcb1ec..92cb526 100644 --- a/test/test_examples_util.rb +++ b/test/test_examples_util.rb @@ -44,6 +44,16 @@ module ValidationExamples end end + def self.validation_get(uri, accept_header='application/rdf+xml') + if $test_case + #puts "getting "+uri+","+accept_header + $test_case.get uri,nil,'HTTP_ACCEPT' => accept_header + return wait($test_case.last_response.body) + else + return OpenTox::RestClientWrapper.get(File.join(@@config[:services]["opentox-validation"],uri),{:accept => accept_header}) + end + end + def self.wait(uri) if OpenTox::Utils.task_uri?(uri) task = OpenTox::Task.find(uri) @@ -54,6 +64,85 @@ module ValidationExamples uri end + def self.verify_crossvalidation(val_yaml) + + val = YAML.load(val_yaml) + puts val.inspect + + assert_integer val["random_seed".to_sym],nil,nil,"random_seed" + assert_boolean val["stratified".to_sym],"stratified" + assert_integer val["num_folds".to_sym],0,1000,"num_folds" + num_folds = val["num_folds".to_sym].to_i + + validations = val["validations".to_sym] + assert_int_equal(num_folds, validations.size, "num_folds != validations.size") + end + + def self.verify_validation(val_yaml) + + val = YAML.load(val_yaml) + + puts val.inspect + assert_integer val["num_instances".to_sym],0,1000,"num_instances" + num_instances = val["num_instances".to_sym].to_i + + assert_integer val["num_unpredicted".to_sym],0,num_instances,"num_unpredicted" + num_unpredicted = val["num_unpredicted".to_sym].to_i + assert_float val["percent_unpredicted".to_sym],0,100 + assert_float_equal(val["percent_unpredicted".to_sym].to_f,100*num_unpredicted/num_instances.to_f,"percent_unpredicted") + + assert_integer val["num_without_class".to_sym],0,num_instances,"num_without_class" + num_without_class = val["num_without_class".to_sym].to_i + assert_float val["percent_without_class".to_sym],0,100 + assert_float_equal(val["percent_without_class".to_sym].to_f,100*num_without_class/num_instances.to_f,"percent_without_class") + + class_stats = val["classification_statistics".to_sym] + if class_stats + class_value_stats = class_stats["class_value_statistics".to_sym] + class_values = [] + class_value_stats.each do |cvs| + class_values << cvs["class_value".to_sym] + end + puts class_values.inspect + + confusion_matrix = class_stats["confusion_matrix".to_sym] + confusion_matrix_cells = confusion_matrix["confusion_matrix_cell".to_sym] + predictions = 0 + confusion_matrix_cells.each do |confusion_matrix_cell| + predictions += confusion_matrix_cell["confusion_matrix_value".to_sym].to_i + end + assert_int_equal(predictions, num_instances-num_unpredicted) + else + regr_stats = val["regression_statistics".to_sym] + assert regr_stats!=nil + end + end + + private + def self.assert_int_equal(val1,val2,msg_suffix=nil) + raise msg_suffix.to_s+" not equal: "+val1.to_s+" != "+val2.to_s unless val1==val2 + end + + def self.assert_float_equal(val1,val2,msg_suffix=nil,epsilon=0.0001) + raise msg_suffix.to_s+" not equal: "+val1.to_s+" != "+val2.to_s+", diff:"+(val1-val2).abs.to_s unless (val1-val2).abs<epsilon + end + + def self.assert_boolean(bool_val,prop=nil) + raise "'"+bool_val.to_s+"' not an boolean "+prop.to_s unless bool_val.to_s=="true" or bool_val.to_s=="false" + end + + def self.assert_integer(string_val, min=nil, max=nil, prop=nil) + raise "'"+string_val.to_s+"' not an integer "+prop.to_s unless string_val.to_i.to_s==string_val.to_s + raise unless string_val.to_i>=min if min!=nil + raise unless string_val.to_i<=max if max!=nil + end + + def self.assert_float(string_val, min=nil, max=nil) + raise string_val.to_s+" not a float (!="+string_val.to_f.to_s+")" unless (string_val.to_f.to_s==string_val.to_s || (string_val.to_f.to_s==(string_val.to_s+".0"))) + raise unless string_val.to_f>=min if min!=nil + raise unless string_val.to_f<=max if max!=nil + end + end class ValidationExample @@ -118,6 +207,16 @@ module ValidationExamples end end + def verify_yaml + if @validation_uri =~ /crossvalidation/ + Util.verify_crossvalidation(Util.validation_get("crossvalidation/"+@validation_uri.split("/")[-1],'text/x-yaml')) + Util.validation_get("crossvalidation/"+@validation_uri.split("/")[-1]+"/statistics",'text/x-yaml') + Util.verify_validation(Util.validation_get("crossvalidation/"+@validation_uri.split("/")[-1]+"/statistics",'text/x-yaml')) + else + Util.verify_validation(Util.validation_get(@validation_uri.split("/")[-1],'text/x-yaml')) + end + end + def title self.class.humanize end diff --git a/validation/validation_application.rb b/validation/validation_application.rb index 9abf2d4..7328077 100644 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -8,9 +8,9 @@ require 'lib/merge.rb' get '/crossvalidation/?' do LOGGER.info "list all crossvalidations" - content_type "text/uri-list" - Validation::Crossvalidation.all(params).collect{ |d| url_for("/crossvalidation/", :full) + d.id.to_s }.join("\n") + params.each{ |k,v| halt 400,"no crossvalidation-attribute: "+k.to_s unless Validation::Crossvalidation.column_names.include?(k.gsub(/_like$/,"")) } + Validation::Crossvalidation.find(:all, :conditions => params).collect{ |d| url_for("/crossvalidation/", :full) + d.id.to_s }.join("\n") end post '/crossvalidation/loo/?' do @@ -61,7 +61,7 @@ get '/crossvalidation/:id/validations' do halt 404, "Crossvalidation '#{params[:id]}' not found." end content_type "text/uri-list" - Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ).collect{ |v| v.uri.to_s }.join("\n")+"\n" + Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ).collect{ |v| v.validation_uri.to_s }.join("\n")+"\n" end @@ -74,7 +74,7 @@ get '/crossvalidation/:id/statistics' do end Lib::MergeObjects.register_merge_attributes( Validation::Validation, - Lib::VAL_MERGE_AVG,Lib::VAL_MERGE_SUM,Lib::VAL_MERGE_GENERAL-[:uri]) unless + Lib::VAL_MERGE_AVG,Lib::VAL_MERGE_SUM,Lib::VAL_MERGE_GENERAL-[:validation_uri]) unless Lib::MergeObjects.merge_attributes_registered?(Validation::Validation) v = Lib::MergeObjects.merge_array_objects( Validation::Validation.find( :all, :conditions => { :crossvalidation_id => params[:id] } ) ) @@ -103,7 +103,7 @@ post '/crossvalidation/?' do cv.create_cv_datasets( params[:prediction_feature] ) cv.perform_cv( params[:algorithm_params]) content_type "text/uri-list" - cv.uri + cv.crossvalidation_uri end halt 202,task_uri end @@ -115,7 +115,8 @@ end get '/?' do LOGGER.info "list all validations" content_type "text/uri-list" - Validation::Validation.all(params).collect{ |d| url_for("/", :full) + d.id.to_s }.join("\n") + params.each{ |k,v| halt 400,"no validation-attribute: "+k.to_s unless Validation::Validation.column_names.include?(k.gsub(/_like$/,"")) } + Validation::Validation.find(:all, :conditions => params).collect{ |d| url_for("/", :full) + d.id.to_s }.join("\n") end get '/:id' do @@ -184,7 +185,7 @@ post '/training_test_split' do :algorithm_uri => params[:algorithm_uri] v.validate_algorithm( params[:algorithm_params]) content_type "text/uri-list" - v.uri + v.validation_uri end halt 202,task_uri end @@ -210,7 +211,7 @@ post '/create_validation' do v = Validation::Validation.new params v.compute_validation_stats() content_type "text/uri-list" - v.uri + v.validation_uri end halt 202,task_uri end diff --git a/validation/validation_format.rb b/validation/validation_format.rb index 97d9eed..f2e3c50 100644 --- a/validation/validation_format.rb +++ b/validation/validation_format.rb @@ -1,15 +1,6 @@ require "lib/rdf_provider.rb" -class String - def convert_underscore - gsub(/_./) do |m| - m.gsub!(/^_/,"") - m.upcase - end - end -end - module Validation @@ -33,19 +24,17 @@ module Validation cv[p] = self.send(p) end # replace crossvalidation id with uri - cv[:crossvalidation_uri] = $sinatra.url_for("/crossvalidation/"+cv[:crossvalidation_id].to_s,:full) if cv[:crossvalidation_id] h[:crossvalidation_info] = cv end if classification_statistics - class_stats = ((classification_statistics.is_a?(Hash)) ? classification_statistics : YAML.load(classification_statistics.to_s)) clazz = {} - Lib::VAL_CLASS_PROPS_SINGLE.each{ |p| clazz[p] = class_stats[p] } + Lib::VAL_CLASS_PROPS_SINGLE.each{ |p| clazz[p] = classification_statistics[p] } # transpose results per class class_values = {} Lib::VAL_CLASS_PROPS_PER_CLASS.each do |p| - $sinatra.halt 500, "missing classification statitstics: "+p.to_s+" "+class_stats.inspect unless class_stats[p] - class_stats[p].each do |class_value, property_value| + $sinatra.halt 500, "missing classification statitstics: "+p.to_s+" "+classification_statistics.inspect unless classification_statistics[p] + classification_statistics[p].each do |class_value, property_value| class_values[class_value] = {:class_value => class_value} unless class_values.has_key?(class_value) map = class_values[class_value] map[p] = property_value @@ -55,8 +44,8 @@ module Validation #converting confusion matrix cells = [] - $sinatra.halt 500,"confusion matrix missing" unless class_stats[:confusion_matrix]!=nil - class_stats[:confusion_matrix].each do |k,v| + $sinatra.halt 500,"confusion matrix missing" unless classification_statistics[:confusion_matrix]!=nil + classification_statistics[:confusion_matrix].each do |k,v| cell = {} # key in confusion matrix is map with predicted and actual attribute k.each{ |kk,vv| cell[kk] = vv } @@ -75,21 +64,15 @@ module Validation return h end - # build hash structure and return with to_yaml - def to_yaml - get_content_as_hash.to_yaml - #super.to_yaml - end - def rdf_title "Validation" end def uri - @uri + validation_uri end - @@literals = [ :created_at, :real_runtime, :num_instances, :num_without_class, + LITERALS = [ :created_at, :real_runtime, :num_instances, :num_without_class, :percent_without_class, :num_unpredicted, :percent_unpredicted, :crossvalidation_fold, #:crossvalidation_id, :num_correct, :num_incorrect, :percent_correct, :percent_incorrect, @@ -102,9 +85,9 @@ module Validation :target_variance_predicted, :mean_absolute_error, :r_square, :class_value, :confusion_matrix_actual, :confusion_matrix_predicted ] - @@literal_names = {:created_at => OT["date"] } - - @@object_properties = { :model_uri => OT['validationModel'], :training_dataset_uri => OT['validationTrainingDataset'], :algorithm_uri => OT['validationAlgorithm'], + LITERAL_NAMES = {:created_at => OT["date"] } + + OBJECT_PROPERTIES = { :model_uri => OT['validationModel'], :training_dataset_uri => OT['validationTrainingDataset'], :algorithm_uri => OT['validationAlgorithm'], :prediction_feature => OT['predictedFeature'], :test_dataset_uri => OT['validationTestDataset'], :test_target_dataset_uri => OT['validationTestTargetDataset'], :prediction_dataset_uri => OT['validationPredictionDataset'], :crossvalidation_info => OT['hasValidationInfo'], :crossvalidation_uri => OT['validationCrossvalidation'], @@ -114,37 +97,11 @@ module Validation #:confusion_matrix_actual => OT['confusionMatrixActual'], :confusion_matrix_predicted => OT['confusionMatrixPredicted'] } - @@classes = { :crossvalidation_info => OT['CrossvalidationInfo'], :classification_statistics => OT['ClassificationStatistics'], + CLASSES = { :crossvalidation_info => OT['CrossvalidationInfo'], :classification_statistics => OT['ClassificationStatistics'], :regression_statistics => OT['RegresssionStatistics'], :class_value_statistics => OT['ClassValueStatistics'], :confusion_matrix => OT['ConfusionMatrix'], :confusion_matrix_cell => OT['ConfusionMatrixCell']} - def literal?( prop ) - @@literals.index( prop ) != nil - end - - def literal_name( prop ) - if @@literal_names.has_key?(prop) - @@literal_names[prop] - else - OT[prop.to_s.convert_underscore] - end - end - - def object_property?( prop ) - @@object_properties.has_key?( prop ) - end - - def object_property_name( prop ) - return @@object_properties[ prop ] - end - - def class?(prop) - @@classes.has_key?( prop ) - end - - def class_name( prop ) - return @@classes[ prop ] - end + IGNORE = [ :id, :validation_uri, :crossvalidation_id ] end @@ -157,58 +114,28 @@ module Validation v = [] Validation.find( :all, :conditions => { :crossvalidation_id => self.id } ).each do |val| - v.push( val.uri.to_s ) + v.push( val.validation_uri.to_s ) end h[:validations] = v h end - - def to_yaml - get_content_as_hash.to_yaml + + def uri + crossvalidation_uri end def rdf_title "Crossvalidation" end - def uri - @uri - end + LITERALS = [ :created_at, :stratified, :num_folds, :random_seed ] - @@literals = [ :created_at, :stratified, :num_folds, :random_seed ] + LITERAL_NAMES = {:created_at => OT["date"] } - @@literal_names = {:created_at => OT["date"] } - - @@object_properties = { :dataset_uri => OT['crossvalidationDataset'], :algorithm_uri => OT['crossvalidationAlgorithm'], + OBJECT_PROPERTIES = { :dataset_uri => OT['crossvalidationDataset'], :algorithm_uri => OT['crossvalidationAlgorithm'], :validations => OT['crossvalidationValidation'] } - @@classes = {} - - def literal?( prop ) - @@literals.index( prop ) != nil - end - - def literal_name( prop ) - if @@literal_names.has_key?(prop) - @@literal_names[prop] - else - OT[prop.to_s.convert_underscore] - end - end + CLASSES = {} - def object_property?( prop ) - @@object_properties.has_key?( prop ) - end - - def object_property_name( prop ) - return @@object_properties[ prop ] - end - - def class?(prop) - @@classes.has_key?( prop ) - end - - def class_name( prop ) - return @@classes[ prop ] - end + IGNORE = [ :id, :crossvalidation_uri ] end end diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 0b6aee9..b2ad83b 100644 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -37,7 +37,7 @@ module Validation # constructs a validation object, Rsets id und uri def initialize( params={} ) $sinatra.halt 500,"do not set id manually" if params[:id] - $sinatra.halt 500,"do not set uri manually" if params[:uri] + $sinatra.halt 500,"do not set uri manually" if params[:validation_uri] super params self.save raise "internal error, validation-id not set "+to_yaml if self.id==nil @@ -45,10 +45,6 @@ module Validation self.save end - def uri - self.validation_uri - end - # deletes a validation # PENDING: model and referenced datasets are deleted as well, keep it that way? def delete @@ -139,9 +135,9 @@ module Validation self.test_dataset_uri, self.test_target_dataset_uri, self.prediction_feature, self.prediction_dataset_uri, model.predictedVariables ) if prediction.classification? - self.attributes = { :classification_statistics => prediction.compute_stats.to_yaml } + self.attributes = { :classification_statistics => prediction.compute_stats } else - self.attributes = { :regression_statistics => prediction.compute_stats.to_yaml } + self.attributes = { :regression_statistics => prediction.compute_stats } end self.attributes = { :num_instances => prediction.num_instances, @@ -159,7 +155,7 @@ module Validation def initialize( params={} ) $sinatra.halt 500,"do not set id manually" if params[:id] - $sinatra.halt 500,"do not set uri manually" if params[:uri] + $sinatra.halt 500,"do not set uri manually" if params[:crossvalidation_uri] params[:num_folds] = 10 if params[:num_folds]==nil params[:random_seed] = 1 if params[:random_seed]==nil @@ -171,10 +167,6 @@ module Validation self.save end - def uri - self.crossvalidation_uri - end - # deletes a crossvalidation, all validations are deleted as well def delete Validation.all(:crossvalidation_id => self.id).each{ |v| v.delete } @@ -225,7 +217,7 @@ module Validation :test_dataset_uri => v.test_dataset_uri, :algorithm_uri => self.algorithm_uri end - LOGGER.debug "copyied dataset uris from cv "+cv.uri.to_s + LOGGER.debug "copied dataset uris from cv "+cv.crossvalidation_uri.to_s return true end diff --git a/validation/validation_test.rb b/validation/validation_test.rb index 9b9780b..65168ca 100644 --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -7,7 +7,7 @@ require 'test/unit' require 'rack/test' require 'lib/test_util.rb' require 'test/test_examples.rb' -LOGGER = Logger.new(STDOUT) +LOGGER = MyLogger.new(STDOUT) LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S " class ValidationTest < Test::Unit::TestCase @@ -17,33 +17,38 @@ class ValidationTest < Test::Unit::TestCase def test_it $test_case = self - # get "/crossvalidation/1/statistics" - # puts last_response.body + #get "?test_dataset_uri_like=92",nil,'HTTP_ACCEPT' => "application/rdf+xml" + #puts last_response.body # post "/test_validation",:select=>"6d" #,:report=>"yes,please" # puts last_response.body - run_test("1b") + #run_test("1b") #, "http://localhost/validation/321") + #run_test("3b", "http://localhost/validation/crossvalidation/1") #puts Nightly.build_nightly("1", false) #prepare_examples - #do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE + do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE end def app Sinatra::Application end - def run_test(select) + def run_test(select, validation_uri=nil) validationExamples = ValidationExamples.select(select) validationExamples.each do |vv| vv.each do |v| ex = v.new - ex.upload_files - ex.check_requirements - ex.validate - LOGGER.debug "validation done "+ex.validation_uri.to_s + ex.validation_uri = validation_uri + unless ex.validation_uri + ex.upload_files + ex.check_requirements + ex.validate + LOGGER.debug "validation done "+ex.validation_uri.to_s + end + ex.verify_yaml ex.report end end |