added nightly report

2 files changed, 317 insertions, 0 deletions

diff --git a/nightly/nightly.rb b/nightly/nightly.rb
new file mode 100644
index 0000000..de56d01
--- /dev/null
+++ b/nightly/nightly.rb
@@ -0,0 +1,295 @@
+
+
+class Nightly
+  
+  NIGHTLY_REP_DIR = File.join(FileUtils.pwd,"nightly")
+  NIGHTLY_REPORT_XML = "nightly_report.xml"
+  NIGHTLY_REPORT_HTML = "nightly_report.html"
+  
+  def self.get_nightly
+    if File.exist?(File.join(NIGHTLY_REP_DIR,NIGHTLY_REPORT_HTML))
+      return File.new(File.join(NIGHTLY_REP_DIR,NIGHTLY_REPORT_HTML))
+    else
+      return "Nightly report not available, try again later"
+    end
+  end
+  
+  def self.build_nightly
+    #OpenTox::Task.as_task do
+      LOGGER.info("Building nightly report.")
+      
+      benchmarks = [ HamsterTrainingTestBenchmark.new, MiniRegressionBenchmark.new ]
+      
+      running = []
+      report = Reports::XMLReport.new("Nightly Validation", Time.now.strftime("Created at %m.%d.%Y - %H:%M"))
+      benchmarks.each do |b|
+        running << b.class.to_s+b.object_id.to_s
+        Thread.new do
+          begin
+            b.build
+          ensure
+            running.delete(b.class.to_s+b.object_id.to_s)
+          end
+        end
+      end
+      wait = 0
+      while running.size>0
+        LOGGER.debug "Nighlty report waiting for "+running.inspect if wait%10==0
+        wait += 1
+        sleep 1
+      end
+      
+      section_about = report.add_section(report.get_root_element, "About this report")
+      report.add_paragraph(section_about,
+        "This a opentox internal test report. Its purpose is to maintain interoperability between the OT validation web service "+
+        "and other OT web services. Please email to guetlein@informatik.uni-freiburg.de if you wish your service/test case to be added.")
+      
+      benchmarks.each do |b|  
+         section = report.add_section(report.get_root_element, b.title)
+         
+         section_about = report.add_section(section, "Info")
+         b.info.each{|i| report.add_paragraph(section_about,i)}
+         info_table = b.info_table
+         report.add_table(section_about, b.info_table_title, info_table) if info_table
+         
+         section_results = report.add_section(section, "Results")
+         report.add_table(section_results, b.result_table_title, b.result_table)
+         
+         section_errors = report.add_section(section, "Errors")
+         
+         if b.errors and b.errors.size>0
+           b.errors.each do |k,v|
+             elem = report.add_section(section_errors,k)
+             report.add_paragraph(elem,v,true)
+           end
+         else
+           report.add_paragraph(section_errors,"no errors occured")
+         end
+       
+      end
+      
+      report.write_to(File.new(File.join(NIGHTLY_REP_DIR,NIGHTLY_REPORT_XML), "w"))
+      Reports::ReportFormat.format_report_to_html(NIGHTLY_REP_DIR,
+        NIGHTLY_REPORT_XML, 
+        NIGHTLY_REPORT_HTML, 
+        "http://opentox.informatik.uni-freiburg.de/simple_ot_stylesheet.css")
+        #"http://www.opentox.org/portal_css/Opentox%20Theme/base-cachekey7442.css")
+        #"http://apps.ideaconsult.net:8080/ToxPredict/style/global.css")
+      
+      LOGGER.info("Nightly report completed")
+      return "Nightly report completed"
+    #end
+  end
+  
+  class ValidationBenchmark
+    
+    def info_table_title
+      return title
+    end
+    
+    def info_table
+      return nil
+    end
+    
+    def result_table_title
+      return "Validation results"
+    end
+    
+    def result_table
+      raise "no comparables" unless @comparables
+      raise "no validations" unless @validations
+      raise "no reports" unless @reports
+      t = []
+      row = [comparable_nice_name, "validation", "report"]
+      t << row
+      (0..@comparables.size-1).each do |i|
+        row = [ @comparables[i], @validations[i], @reports[i] ]
+        t << row
+      end
+      t
+    end
+  end
+  
+  class TrainingTestValidationBenchmark < ValidationBenchmark
+    
+    def info
+      [ training_test_info ]
+    end
+    
+    def training_test_info
+      "This is a training test set validation. It builds a model with an algorithm and the training dataset. "+
+      "The model is used to predict the test dataset. Evaluation is done by comparing the model predictions "+
+      "to the actual test values (in the test target dataset)."
+    end
+    
+    def info_table_title
+      "Validation params"
+    end
+    
+    def comparable_nice_name
+      return "algorithm"
+    end
+    
+    def info_table
+      t = []
+      t << ["param", "uri"]
+      t << ["training_dataset_uri", @train_data]
+      t << ["test_dataset_uri", @test_data]
+      t << ["test_target_dataset_uri", @test_class_data] if @test_class_data
+      t << ["prediction_feature", @pred_feature]
+      count = 1
+      @algs.each do |alg|
+        t << ["algorithm_uri"+" ["+count.to_s+"]", alg]
+        count += 1
+      end
+      t
+    end
+    
+    def errors
+      @errors
+    end
+    
+    
+    def build()
+      raise "no algs" unless @algs
+      raise "no train data" unless @train_data
+      raise "no test data" unless @test_data
+      raise "no pred feature" unless @pred_feature
+      
+      @comparables = @algs
+      @validations = Array.new(@comparables.size)
+      @reports = Array.new(@comparables.size)
+      @errors = {}
+#      LOGGER.info "train-data: "+@train_data.to_s
+#      LOGGER.info "test-data: "+@test_data.to_s
+#      LOGGER.info "test-class-data: "+@test_class_data.to_s
+      
+      running = []
+      (0..@comparables.size-1).each do |i|
+        
+        Thread.new do 
+          running << @comparables[i]+i.to_s
+          begin
+            LOGGER.info "validate: "+@algs[i].to_s
+            @validations[i] = Util.validate_alg(@train_data, @test_data, @test_class_data,
+              @algs[i], URI.decode(@pred_feature), @alg_params[i])
+              
+            begin
+              LOGGER.info "building validation-report"
+              @reports[i] = Util.create_report(@validations[i])
+            rescue => ex
+              LOGGER.error "validation-report error: "+ex.message
+              @reports[i] = "error"
+            end
+            
+          rescue => ex
+            LOGGER.error "validation error: "+ex.message
+            key = "Error validating "+@comparables[i].to_s
+            @validations[i] = key+" (see below)"
+            @errors[key] = ex.message
+          ensure
+            running.delete(@comparables[i]+i.to_s)
+          end
+        end
+      end
+      wait = 0
+      while running.size>0
+        LOGGER.debug self.class.to_s+" waiting for "+running.inspect if wait%5==0
+        wait += 1
+        sleep 1
+      end
+    end
+  end
+  
+  class MiniRegressionBenchmark < TrainingTestValidationBenchmark
+    
+    
+    def title
+      "Training test set validation, regression"
+    end
+    
+    def info
+      res = [ "A very small regression task, using the training dataset as test set." ] + super
+      return res
+    end
+    
+    def build()
+      @algs = [ "http://opentox.ntua.gr:3000/algorithm/mlr", 
+        "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNregression"]
+      @alg_params = [nil, nil]
+      #@pred_feature = "http://apps.ideaconsult.net:8080/ambit2/feature/22200"
+      #@train_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/54"
+      #@test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55"
+      
+      @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
+      @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
+      @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
+      super
+    end
+  end
+ 
+  class HamsterTrainingTestBenchmark < TrainingTestValidationBenchmark
+    
+    @@dataset_service = @@config[:services]["opentox-dataset"]
+    @@file=File.new("data/hamster_carcinogenicity.yaml","r")
+    @@file_type="text/x-yaml"
+    @@lazar_server = @@config[:services]["opentox-algorithm"]
+    
+    def title()
+      "Training test set validation, binary classification"
+    end
+    
+    def info
+      res = [ "A simple binary classification task using the hamster carcinogenicity dataset." ] + super
+      return res
+    end
+    
+    def build()
+      @algs = [File.join(@@lazar_server,"lazar")]
+      @alg_params = ["feature_generation_uri="+File.join(@@lazar_server,"fminer")]
+      @pred_feature = "http://localhost/toxmodel/feature%23Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+
+      LOGGER.debug "pepare hamster datasets"
+      @test_class_data = Util.upload_dataset(@@dataset_service, @@file, @@file_type).chomp("\n")
+      split = Util.split_dataset(@test_class_data, URI.decode(@pred_feature), 0.9, 1)
+      @train_data = split[0].to_s
+      @test_data = split[1].to_s
+      super
+    end
+  end
+
+
+  class Util
+    @@validation_service = @@config[:services]["opentox-validation"]
+    
+    def self.upload_dataset(dataset_service, file, file_type)
+      raise "File not found: "+file.path.to_s unless File.exist?(file.path)
+      data = File.read(file.path)
+      data_uri = OpenTox::RestClientWrapper.post dataset_service, data, {:content_type => file_type}, true
+      #data_uri = OpenTox::Task.find(data_uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(data_uri)
+      return data_uri
+    end
+    
+    def self.split_dataset(data_uri, feature, split_ratio, random_seed)
+      res = OpenTox::RestClientWrapper.post @@validation_service+'/plain_training_test_split', { :dataset_uri => data_uri, :prediction_feature=>feature, :split_ratio=>split_ratio, :random_seed=>random_seed}
+      return res.split("\n")
+    end
+    
+    def self.validate_alg(train_data, test_data, test_class_data, alg, feature, alg_params)
+      uri = OpenTox::RestClientWrapper.post @@validation_service, { :training_dataset_uri => train_data, :test_dataset_uri => test_data, 
+        :test_target_dataset_uri => test_class_data,
+        :algorithm_uri => alg, :prediction_feature => feature, :algorithm_params => alg_params }, nil, true
+      #LOGGER.info "waiting for validation "+uri.to_s
+      #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
+      #LOGGER.info "validaiton done "+uri.to_s
+      return uri
+    end
+    
+    def self.create_report(validation)
+      uri = OpenTox::RestClientWrapper.post @@validation_service+"/report/validation", { :validation_uris => validation }, nil, true
+      #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
+      return uri
+    end
+  end
+  
+end
\ No newline at end of file
diff --git a/nightly/nightly_application.rb b/nightly/nightly_application.rb
new file mode 100644
index 0000000..c6fa887
--- /dev/null
+++ b/nightly/nightly_application.rb
@@ -0,0 +1,22 @@
+require "nightly/nightly.rb"
+
+get '/build_nightly/?' do
+  Nightly.build_nightly()
+end
+
+get '/css_style_sheet/?' do
+  perform do |rs|
+    "@import \""+params[:css_style_sheet]+"\";"
+  end
+end
+
+get '/nightly/?' do
+  LOGGER.info "get nightly"
+  content_type "text/html"
+  rep = Nightly.get_nightly
+  if rep.is_a?(File)
+    result = body(File.new("/home/martin/software/sinatra/opentox-validation/nightly_report.html"))
+  else
+    result = rep
+  end
+end