From 3a11ba2918795821600b7113d0758415718d263a Mon Sep 17 00:00:00 2001 From: gebele Date: Mon, 11 Jun 2018 12:46:06 +0200 Subject: combine gui with rest --- application.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 895b0c2..04e5352 100644 --- a/application.rb +++ b/application.rb @@ -26,10 +26,27 @@ error do haml :error end -get '/?' do - redirect to('/predict') +# https://github.com/britg/sinatra-cross_origin#responding-to-options +options "*" do + response.headers["Allow"] = "HEAD,GET,PUT,POST,DELETE,OPTIONS" + response.headers["Access-Control-Allow-Headers"] = "X-Requested-With, X-HTTP-Method-Override, Content-Type, Cache-Control, Accept" + 200 end +[ + "aa.rb", + "api.rb", + "compound.rb", + "dataset.rb", + "feature.rb", + "model.rb", + "nanoparticle.rb", + "report.rb", + "substance.rb", + "swagger.rb", + "validation.rb" +].each{ |f| require_relative "./lib/#{f}" } + get '/predict/?' do @models = OpenTox::Model::Validation.all @models = @models.delete_if{|m| m.model.name =~ /\b(Net cell association)\b/} -- cgit v1.2.3 From f6f186de7144bfbddc508e1330f334d3ee7f2005 Mon Sep 17 00:00:00 2001 From: gebele Date: Tue, 12 Jun 2018 10:30:14 +0200 Subject: adjust link;add logo route --- application.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 04e5352..8d0429f 100644 --- a/application.rb +++ b/application.rb @@ -298,3 +298,6 @@ get '/style.css' do scss :style end +get '/IST_logo_s.png' do + redirect to('/images/IST_logo_s.png') +end -- cgit v1.2.3 From ba65db7e62264b94a7303681e30db4ac415b4822 Mon Sep 17 00:00:00 2001 From: gebele Date: Tue, 12 Jun 2018 16:23:10 +0200 Subject: fixed api content types --- application.rb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 8d0429f..96e564b 100644 --- a/application.rb +++ b/application.rb @@ -14,7 +14,24 @@ configure :development do end before do - @version = File.read("VERSION").chomp + paths = [ + "/aa", + "/api", + "/compound", + "/dataset", + "/feature", + "/model", + "/nanoparticle", + "/report", + "/substance", + "/swagger", + "/validation"] + if paths.include?(request.path) + @accept = request.env['HTTP_ACCEPT'] + response['Content-Type'] = @accept + else + @version = File.read("VERSION").chomp + end end not_found do -- cgit v1.2.3 From 395506ca3fe4daa5689fd197e57f7ab944beb1d7 Mon Sep 17 00:00:00 2001 From: gebele Date: Thu, 28 Jun 2018 14:35:20 +0000 Subject: updated API and request path handling --- application.rb | 53 +++++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 26 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 96e564b..ebcb0af 100644 --- a/application.rb +++ b/application.rb @@ -2,6 +2,20 @@ require 'rdiscount' require_relative 'qmrf_report.rb' include OpenTox +[ + "aa.rb", + "api.rb", + "compound.rb", + "dataset.rb", + "feature.rb", + "model.rb", + "nanoparticle.rb", + "report.rb", + "substance.rb", + "swagger.rb", + "validation.rb" +].each{ |f| require_relative "./lib/#{f}" } + configure :production do $logger = Logger.new(STDOUT) @@ -15,18 +29,19 @@ end before do paths = [ - "/aa", - "/api", - "/compound", - "/dataset", - "/feature", - "/model", - "/nanoparticle", - "/report", - "/substance", - "/swagger", - "/validation"] - if paths.include?(request.path) + "/", + "aa", + "api", + "compound", + "dataset", + "feature", + "model", + "nanoparticle", + "report", + "substance", + "swagger", + "validation"] + if request.path == "/" || paths.include?(request.path.split("/")[1]) @accept = request.env['HTTP_ACCEPT'] response['Content-Type'] = @accept else @@ -50,20 +65,6 @@ options "*" do 200 end -[ - "aa.rb", - "api.rb", - "compound.rb", - "dataset.rb", - "feature.rb", - "model.rb", - "nanoparticle.rb", - "report.rb", - "substance.rb", - "swagger.rb", - "validation.rb" -].each{ |f| require_relative "./lib/#{f}" } - get '/predict/?' do @models = OpenTox::Model::Validation.all @models = @models.delete_if{|m| m.model.name =~ /\b(Net cell association)\b/} -- cgit v1.2.3 From 878f014ec6cc808af99af5045bcc1a1143cab8d9 Mon Sep 17 00:00:00 2001 From: gebele Date: Thu, 5 Jul 2018 10:38:55 +0000 Subject: updated with endpoint list; refined error handling; refined prediction input --- application.rb | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index ebcb0af..a5f9ff2 100644 --- a/application.rb +++ b/application.rb @@ -7,6 +7,7 @@ include OpenTox "api.rb", "compound.rb", "dataset.rb", + "endpoint.rb", "feature.rb", "model.rb", "nanoparticle.rb", @@ -28,12 +29,13 @@ configure :development do end before do - paths = [ + $paths = [ "/", "aa", "api", "compound", "dataset", + "endpoint", "feature", "model", "nanoparticle", @@ -41,7 +43,7 @@ before do "substance", "swagger", "validation"] - if request.path == "/" || paths.include?(request.path.split("/")[1]) + if request.path == "/" || $paths.include?(request.path.split("/")[1]) @accept = request.env['HTTP_ACCEPT'] response['Content-Type'] = @accept else @@ -54,8 +56,14 @@ not_found do end error do - @error = request.env['sinatra.error'] - haml :error + if request.path == "/" || $paths.include?(request.path.split("/")[1]) + @accept = request.env['HTTP_ACCEPT'] + response['Content-Type'] = @accept + @accept == "text/plain" ? request.env['sinatra.error'] : request.env['sinatra.error'].to_json + else + @error = request.env['sinatra.error'] + haml :error + end end # https://github.com/britg/sinatra-cross_origin#responding-to-options -- cgit v1.2.3 From b3ed4f0dbf47ed4986b7ff745324fbde20123631 Mon Sep 17 00:00:00 2001 From: gebele Date: Mon, 27 Aug 2018 11:33:10 +0000 Subject: updated to bs 4 --- application.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index a5f9ff2..f2d2e32 100644 --- a/application.rb +++ b/application.rb @@ -316,7 +316,7 @@ end get '/faq' do @faq = RDiscount.new(File.read("FAQ.md")).to_html - haml :faq, :layout => false + haml :faq#, :layout => false end get '/style.css' do @@ -324,6 +324,11 @@ get '/style.css' do scss :style end +get '/swagger-ui.css' do + headers 'Content-Type' => 'text/css; charset=utf-8' + scss :style +end + get '/IST_logo_s.png' do redirect to('/images/IST_logo_s.png') end -- cgit v1.2.3 From 9750e0309500259e9a56e267ce87984fb5bb5e53 Mon Sep 17 00:00:00 2001 From: gebele Date: Mon, 26 Nov 2018 15:29:26 +0000 Subject: clean out; better response codes; prepare for batch --- application.rb | 504 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 330 insertions(+), 174 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index f2d2e32..dd62fb6 100644 --- a/application.rb +++ b/application.rb @@ -1,16 +1,18 @@ require 'rdiscount' require_relative 'qmrf_report.rb' +require_relative 'task.rb' +require_relative 'prediction.rb' +require_relative 'batch.rb' +require_relative 'helper.rb' include OpenTox [ - "aa.rb", "api.rb", "compound.rb", "dataset.rb", "endpoint.rb", "feature.rb", "model.rb", - "nanoparticle.rb", "report.rb", "substance.rb", "swagger.rb", @@ -18,27 +20,37 @@ include OpenTox ].each{ |f| require_relative "./lib/#{f}" } -configure :production do - $logger = Logger.new(STDOUT) - enable :reloader -end - -configure :development do +configure :production, :development do + STDOUT.sync = true $logger = Logger.new(STDOUT) + $logger.level = Logger::DEBUG enable :reloader + also_reload './helper.rb' + also_reload './prediction.rb' + also_reload './batch.rb' + [ + "api.rb", + "compound.rb", + "dataset.rb", + "endpoint.rb", + "feature.rb", + "model.rb", + "report.rb", + "substance.rb", + "swagger.rb", + "validation.rb" + ].each{ |f| also_reload "./lib/#{f}" } end before do $paths = [ "/", - "aa", "api", "compound", "dataset", "endpoint", "feature", "model", - "nanoparticle", "report", "substance", "swagger", @@ -92,21 +104,18 @@ get '/predict/modeldetails/:model' do return haml :model_details, :layout=> false, :locals => {:model => model, :crossvalidations => crossvalidations} end -# get individual compound details -get '/prediction/:neighbor/details/?' do - @compound = OpenTox::Compound.find params[:neighbor] - @smiles = @compound.smiles - begin - @names = @compound.names.nil? ? "No names for this compound available." : @compound.names - rescue - @names = "No names for this compound available." - end - @inchi = @compound.inchi.gsub("InChI=", "") - - haml :details, :layout => false +get "/predict/report/:id/?" do + prediction_model = Model::Validation.find params[:id] + bad_request_error "model with id: '#{params[:id]}' not found." unless prediction_model + report = qmrf_report params[:id] + # output + t = Tempfile.new + t << report.to_xml + name = prediction_model.species.sub(/\s/,"-")+"-"+prediction_model.endpoint.downcase.sub(/\s/,"-") + send_file t.path, :filename => "QMRF_report_#{name.gsub!(/[^0-9A-Za-z]/, '_')}.xml", :type => "application/xml", :disposition => "attachment" end -get '/jme_help/?' do +get '/predict/jme_help/?' do File.read(File.join('views','jme_help.html')) end @@ -123,190 +132,336 @@ get '/predict/:tmppath/:filename/?' do send_file path, :filename => "lazar_batch_prediction_#{params[:filename]}", :type => "text/csv", :disposition => "attachment" end -post '/predict/?' do +get '/predict/csv/:task/:model/:filename/?' do + response['Content-Type'] = "text/csv" + filename = params[:filename] =~ /\.csv$/ ? params[:filename].gsub(/\.csv$/,"") : params[:filename] + task = Task.find params[:task].to_s + m = Model::Validation.find params[:model].to_s + dataset = Batch.find_by(:name => filename) + @ids = dataset.ids + warnings = dataset.warnings.blank? ? nil : dataset.warnings.join("\n") + unless warnings.nil? + @parse = [] + warnings.split("\n").each do |warning| + if warning =~ /^Cannot/ + smi = warning.split("SMILES compound").last.split("at").first + line = warning.split("SMILES compound").last.split("at line").last.split("of").first.strip.to_i + @parse << "Cannot parse SMILES compound#{smi}at line #{line} of #{dataset.source.split("/").last}\n" + end + end + keys_array = [] + warnings.split("\n").each do |warning| + if warning =~ /^Duplicate/ + text = warning.split("ID").first + numbers = warning.split("ID").last.split("and") + keys_array << numbers.collect{|n| n.strip.to_i} + end + end + @dups = {} + keys_array.each do |keys| + keys.each do |key| + @dups[key] = "Duplicate compound at ID #{keys.join(" and ")}\n" + end + end + end + endpoint = "#{m.endpoint}_(#{m.species})" + tempfile = Tempfile.new + header = task.csv + lines = [] + task.predictions[params[:model]].each_with_index do |hash,idx| + identifier = hash.keys[0] + prediction_id = hash.values[0] + # add duplicate warning at the end of a line if ID matches + if @dups && @dups[idx+1] + if prediction_id.is_a? BSON::ObjectId + if @ids.blank? + lines << "#{idx+1},#{identifier},#{Prediction.find(prediction_id).csv.tr("\n","")},#{@dups[idx+1]}" + else + lines << "#{idx+1},#{@ids[idx]},#{identifier},#{Prediction.find(prediction_id).csv.tr("\n","")},#{@dups[idx+1]}" + end + end + else + if prediction_id.is_a? BSON::ObjectId + if @ids.blank? + lines << "#{idx+1},#{identifier},#{Prediction.find(prediction_id).csv}" + else + lines << "#{idx+1},#{@ids[idx]},#{identifier},#{Prediction.find(prediction_id).csv}" + end + else + if @ids.blank? + lines << "#{idx+1},#{identifier},#{p}\n" + else + lines << "#{idx+1},#{@ids[idx]}#{identifier},#{p}\n" + end + end + end + end + (@parse && !@parse.blank?) ? tempfile.write(header+lines.join("")+"\n"+@parse.join("\n")) : tempfile.write(header+lines.join("")) + #tempfile.write(header+lines.join("")) + tempfile.rewind + send_file tempfile, :filename => "#{Time.now.strftime("%Y-%m-%d")}_lazar_batch_prediction_#{endpoint}_#{filename}.csv", :type => "text/csv", :disposition => "attachment" +end +post '/predict/?' do # process batch prediction if !params[:fileselect].blank? + next if params[:fileselect][:filename] !~ /\.csv$/ - bad_request_error "Please submit a csv file." - end - File.open('tmp/' + params[:fileselect][:filename], "w") do |f| - f.write(params[:fileselect][:tempfile].read) + bad_request_error "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file." end @filename = params[:fileselect][:filename] begin - input = OpenTox::Dataset.from_csv_file File.join("tmp", params[:fileselect][:filename]), true - if input.class == OpenTox::Dataset - dataset = OpenTox::Dataset.find input + File.open('tmp/' + params[:fileselect][:filename], "w") do |f| + f.write(params[:fileselect][:tempfile].read) + end + input = Batch.from_csv_file File.join("tmp", params[:fileselect][:filename]) + $logger.debug "Processing '#{params[:fileselect][:filename]}'" + if input.class == OpenTox::Batch + @dataset = input + @compounds = @dataset.compounds + @identifiers = @dataset.identifiers + @ids = @dataset.ids else + File.delete File.join("tmp", params[:fileselect][:filename]) bad_request_error "Could not serialize file '#{@filename}'." end rescue + File.delete File.join("tmp", params[:fileselect][:filename]) bad_request_error "Could not serialize file '#{@filename}'." end - @compounds = dataset.compounds + if @compounds.size == 0 - message = dataset[:warnings] - dataset.delete + message = @dataset.warnings + @dataset.delete bad_request_error message end - - # for csv export - @batch = {} - # for haml table - @view = {} - - @compounds.each{|c| @view[c] = []} - params[:selection].keys.each do |model_id| - model = OpenTox::Model::Validation.find model_id - @batch[model] = [] - @compounds.each_with_index do |compound,idx| - prediction = model.predict(compound) - @batch[model] << [compound, prediction] - @view[compound] << [model,prediction] - end - end - - @csvhash = {} - @warnings = dataset[:warnings] - dupEntries = {} - delEntries = "" - - # split duplicates and deleted entries - @warnings.each do |w| - substring = w.match(/line .* of/) - unless substring.nil? - delEntries += "\"#{w.sub(/\b(tmp\/)\b/,"")}\"\n" - end - substring = w.match(/rows .* Entries/) - unless substring.nil? - lines = [] - substring[0].split(",").each{|s| lines << s[/\d+/]} - lines.shift - lines.each{|l| dupEntries[l.to_i] = w.split(".").first} - end - end - - @batch.each_with_index do |hash, idx| - @csvhash[idx] = "" - model = hash[0] - # create header - if model.regression? - predAunit = "(#{model.unit})" - predBunit = "(#{model.unit =~ /mmol\/L/ ? "(mol/L)" : "(mg/kg_bw/day)"})" - @csvhash[idx] = "\"ID\",\"Endpoint\",\"Type\",\"Unique SMILES\",\"Prediction #{predAunit}\",\"Prediction #{predBunit}\",\"95% Prediction interval (low) #{predAunit}\",\"95% Prediction interval (high) #{predAunit}\",\"95% Prediction interval (low) #{predBunit}\",\"95% Prediction interval (high) #{predBunit}\",\"inApplicabilityDomain\",\"inTrainningSet\",\"Note\"\n" - else #classification - av = model.prediction_feature.accept_values - probFirst = av[0].capitalize - probLast = av[1].capitalize - @csvhash[idx] = "\"ID\",\"Endpoint\",\"Type\",\"Unique SMILES\",\"Prediction\",\"predProbability#{probFirst}\",\"predProbability#{probLast}\",\"inApplicabilityDomain\",\"inTrainningSet\",\"Note\"\n" - end - values = hash[1] - dupEntries.keys.each{|k| values.insert(k-1, dupEntries[k])}.compact! - - values.each_with_index do |array, id| - type = (model.regression? ? "Regression" : "Classification") - endpoint = "#{model.endpoint.gsub('_', ' ')} (#{model.species})" - if id == 0 - @csvhash[idx] += delEntries unless delEntries.blank? + @models = params[:selection].keys + # for single predictions in batch + @tasks = [] + @models.each{|m| t = Task.new; t.save; @tasks << t} + @predictions = {} + task = Task.run do + @models.each_with_index do |model,idx| + t = @tasks[idx] + m = Model::Validation.find model + type = (m.regression? ? "Regression" : "Classification") + # add header for regression + if type == "Regression" + unit = (type == "Regression") ? "(#{m.unit})" : "" + converted_unit = (type == "Regression") ? "#{m.unit =~ /\b(mmol\/L)\b/ ? "(mg/L)" : "(mg/kg_bw/day)"}" : "" + if @ids.blank? + header = "ID,Input,Endpoint,Unique SMILES,inTrainingSet,Measurements #{unit},Prediction #{unit},Prediction #{converted_unit},"\ + "Prediction Interval Low #{unit},Prediction Interval High #{unit},"\ + "Prediction Interval Low #{converted_unit},Prediction Interval High #{converted_unit},"\ + "inApplicabilityDomain,Note\n" + else + header = "ID,Original ID,Input,Endpoint,Unique SMILES,inTrainingSet,Measurements #{unit},Prediction #{unit},Prediction #{converted_unit},"\ + "Prediction Interval Low #{unit},Prediction Interval High #{unit},"\ + "Prediction Interval Low #{converted_unit},Prediction Interval High #{converted_unit},"\ + "inApplicabilityDomain,Note\n" + end end - unless array.kind_of? String - compound = array[0] - prediction = array[1] - smiles = compound.smiles - - if prediction[:neighbors] - if prediction[:value] - pred = prediction[:value].numeric? ? "#{prediction[:value].delog10.signif(3)}" : prediction[:value] - predA = prediction[:value].numeric? ? "#{prediction[:value].delog10.signif(3)}" : prediction[:value] - predAunit = prediction[:value].numeric? ? "(#{model.unit})" : "" - predB = prediction[:value].numeric? ? "#{compound.mmol_to_mg(prediction[:value].delog10).signif(3)}" : prediction[:value] - predBunit = prediction[:value].numeric? ? "#{model.unit =~ /\b(mmol\/L)\b/ ? "(mg/L)" : "(mg/kg_bw/day)"}" : "" - int = (prediction[:prediction_interval].nil? ? nil : prediction[:prediction_interval]) - intervalLow = (int.nil? ? "" : "#{int[1].delog10.signif(3)}") - intervalHigh = (int.nil? ? "" : "#{int[0].delog10.signif(3)}") - intervalLowMg = (int.nil? ? "" : "#{compound.mmol_to_mg(int[1].delog10).signif(3)}") - intervalHighMg = (int.nil? ? "" : "#{compound.mmol_to_mg(int[0].delog10).signif(3)}") - inApp = "yes" - inT = prediction[:info] =~ /\b(identical)\b/i ? "yes" : "no" - note = prediction[:warnings].join("\n") + ( prediction[:info] ? prediction[:info].sub(/\'.*\'/,"") : "\n" ) - - unless prediction[:probabilities].nil? - av = model.prediction_feature.accept_values - propA = "#{prediction[:probabilities][av[0]].to_f.signif(3)}" - propB = "#{prediction[:probabilities][av[1]].to_f.signif(3)}" - end - else - # no prediction value only one neighbor - inApp = "no" - inT = prediction[:info] =~ /\b(identical)\b/i ? "yes" : "no" - note = prediction[:warnings].join("\n") + ( prediction[:info] ? prediction[:info].sub(/\'.*\'/,"") : "\n" ) - end + # add header for classification + if type == "Classification" + av = m.prediction_feature.accept_values + if @ids.blank? + header = "ID,Input,Endpoint,Unique SMILES,inTrainingSet,Measurements,Prediction,"\ + "predProbability #{av[0]},predProbability #{av[1]},inApplicabilityDomain,Note\n" else - # no prediction value - inApp = "no" - inT = prediction[:info] =~ /\b(identical)\b/i ? "yes" : "no" - note = prediction[:warnings].join("\n") + ( prediction[:info] ? prediction[:info].sub(/\'.*\'/,"") : "\n" ) + header = "ID,Original ID,Input,Endpoint,Unique SMILES,inTrainingSet,Measurements,Prediction,"\ + "predProbability #{av[0]},predProbability #{av[1]},inApplicabilityDomain,Note\n" end - if @warnings - @warnings.each do |w| - note += (w.split(".").first + ".") if /\b(#{Regexp.escape(smiles)})\b/ === w + end + # predict compounds + p = 100.0/@compounds.size + counter = 1 + predictions = [] + @compounds.each_with_index do |cid,idx| + compound = Compound.find cid + if Prediction.where(compound: compound.id, model: m.id).exists? + prediction_object = Prediction.find_by(compound: compound.id, model: m.id) + prediction = prediction_object.prediction + prediction_id = prediction_object.id + # in case prediction object was created by single prediction + if prediction_object.csv.blank? + prediction_object[:csv] = prediction_to_csv(m,compound,prediction) + prediction_object.save + end + # identifier + identifier = @identifiers[idx] + else + prediction = m.predict(compound) + # save prediction object + prediction_object = Prediction.new + prediction_id = prediction_object.id + prediction_object[:compound] = compound.id + prediction_object[:model] = m.id + # add additionally fields for html representation + unless prediction[:value].blank? || type == "Classification" + prediction[:prediction_value] = "#{prediction[:value].delog10.signif(3)} #{unit}" + prediction["converted_prediction_value"] = "#{compound.mmol_to_mg(prediction[:value].delog10).signif(3)} #{converted_unit}" + end + unless prediction[:prediction_interval].blank? + interval = prediction[:prediction_interval] + prediction[:interval] = "#{interval[1].delog10.signif(3)} - #{interval[0].delog10.signif(3)} #{unit}" + prediction[:converted_interval] = "#{compound.mmol_to_mg(interval[1].delog10).signif(3)} - #{compound.mmol_to_mg(interval[0].delog10).signif(3)} #{converted_unit}" end + prediction["unit"] = unit + prediction["converted_unit"] = converted_unit + if prediction[:measurements].is_a?(Array) + prediction["measurements_string"] = (type == "Regression") ? prediction[:measurements].collect{|value| "#{value.delog10.signif(3)} #{unit}"} : prediction[:measurements].join("
") + prediction["converted_measurements"] = prediction[:measurements].collect{|value| "#{compound.mmol_to_mg(value.delog10).signif(3)} #{unit =~ /mmol\/L/ ? "(mg/L)" : "(mg/kg_bw/day)"}"} if type == "Regression" + else + output["measurements_string"] = (type == "Regression") ? "#{prediction[:measurements].delog10.signif(3)} #{unit}}" : prediction[:measurements] + output["converted_measurements"] = "#{compound.mmol_to_mg(prediction[:measurements].delog10).signif(3)} #{(unit =~ /\b(mmol\/L)\b/) ? "(mg/L)" : "(mg/kg_bw/day)"}" if type == "Regression" + end + + # store in prediction_object + prediction_object[:prediction] = prediction + prediction_object[:csv] = prediction_to_csv(m,compound,prediction) + prediction_object.save + + # identifier + identifier = @identifiers[idx] end - else - # string note for duplicates - endpoint = type = smiles = pred = predA = predB = propA = propB = intervalLow = intervalHigh = intervalLowMg = intervalHighMg = inApp = inT = "" - note = array + # collect prediction_object ids with identifier + predictions << {identifier => prediction_id} + t.update_percent((counter*p).ceil > 100 ? 100 : (counter*p).ceil) + counter += 1 end - if model.regression? - @csvhash[idx] += "\"#{id+1}\",\"#{endpoint}\",\"#{type}\",\"#{smiles}\",\"#{predA}\",\"#{predB}\",\"#{intervalLow}\",\"#{intervalHigh}\",\"#{intervalLowMg}\",\"#{intervalHighMg}\",\"#{inApp}\",\"#{inT}\",\"#{note.chomp}\"\n" + # write csv + t[:csv] = header + # write predictions + @predictions["#{model}"] = predictions + # save task + # append predictions as last action otherwise they won't save + # mongoid works with shallow copy via #dup + t[:predictions] = @predictions + t.save + end#models + + end#main task + @pid = task.pid + + #@dataset.delete + #File.delete File.join("tmp", params[:fileselect][:filename]) + return haml :batch + else + # single compound prediction + # validate identifier input + if !params[:identifier].blank? + @identifier = params[:identifier].strip + $logger.debug "input:#{@identifier}" + # get compound from SMILES + @compound = Compound.from_smiles @identifier + bad_request_error "'#{@identifier}' is not a valid SMILES string." if @compound.blank? + + @models = [] + @predictions = [] + @toxtree = false + params[:selection].keys.each do |model_id| + model = Model::Validation.find model_id + @models << model + if Prediction.where(compound: @compound.id, model: model.id).exists? + prediction_object = Prediction.find_by(compound: @compound.id, model: model.id) + prediction = prediction_object.prediction + @predictions << prediction else - @csvhash[idx] += "\"#{id+1}\",\"#{endpoint}\",\"#{type}\",\"#{smiles}\",\"#{pred}\",\"#{propA}\",\"#{propB}\",\"#{inApp}\",\"#{inT}\",\"#{note.chomp}\"\n" + prediction_object = Prediction.new + prediction = model.predict(@compound) + prediction_object[:compound] = @compound.id + prediction_object[:model] = model.id + prediction_object[:prediction] = prediction + prediction_object.save + @predictions << prediction end end - end - t = Tempfile.new - @csvhash.each do |model, csv| - t.write(csv) - t.write("\n") - end - t.rewind - @tmppath = t.path.split("/").last - dataset.delete - File.delete File.join("tmp", params[:fileselect][:filename]) - return haml :batch + haml :prediction + end end +end - # validate identifier input - if !params[:identifier].blank? - @identifier = params[:identifier] - $logger.debug "input:#{@identifier}" - # get compound from SMILES - @compound = Compound.from_smiles @identifier - bad_request_error "'#{@identifier}' is not a valid SMILES string." if @compound.blank? +get '/prediction/task/?' do + if params[:turi] + task = Task.find(params[:turi].to_s) + return JSON.pretty_generate(:percent => task.percent) + elsif params[:predictions] + task = Task.find(params[:predictions]) + pageSize = params[:pageSize].to_i - 1 + pageNumber= params[:pageNumber].to_i - 1 + predictions = task.predictions[params[:model]].collect{|hash| hash.values[0]} + prediction_object = Prediction.find predictions[pageNumber] + prediction = prediction_object.prediction + compound = Compound.find prediction_object.compound + model = Model::Validation.find prediction_object.model + image = compound.svg + smiles = compound.smiles + type = (model.regression? ? "Regression" : "Classification") + html = "" + html += "" + string = "" + html += "#{string}
#{image}
#{smiles}
" + sorter = [] + if prediction[:info] + prediction[:info] = "This compound was part of the training dataset. All information from this compound was "\ + "removed from the training data before the prediction to obtain unbiased results." + sorter << {"Info" => prediction[:info]} + if prediction["measurements_string"].kind_of?(Array) + sorter << {"Measured activity" => "#{prediction["measurements_string"].join(";")}
#{prediction["converted_measurements"].join(";")}"} + else + sorter << {"Measured activity" => "#{prediction["measurements_string"]}
#{prediction["converted_measurements"]}"} + end + end - @models = [] - @predictions = [] - params[:selection].keys.each do |model_id| - model = OpenTox::Model::Validation.find model_id - @models << model - @predictions << model.predict(@compound) + # regression + if prediction[:value] && type == "Regression" + sorter << {"Prediction" => "#{prediction["prediction_value"]}
#{prediction["converted_prediction_value"]}"} + sorter << {"95% Prediction interval" => "#{prediction[:interval]}
#{prediction["converted_interval"]}"} + sorter << {"Warnings" => prediction[:warnings].join("
")} + elsif !prediction[:value] && type == "Regression" + sorter << {"Prediction" => ""} + sorter << {"95% Prediction interval" => ""} + sorter << {"Warnings" => prediction[:warnings].join("
")} + # classification + elsif prediction[:value] && type == "Classification" + sorter << {"Prediction" => prediction[:value]} + sorter << {"Probability" => prediction[:probabilities].collect{|k,v| "#{k}: #{v.signif(3)}"}.join("
")} + elsif !prediction[:value] && type == "Classification" + sorter << {"Prediction" => ""} + sorter << {"Probability" => ""} + #else + sorter << {"Warnings" => prediction[:warnings].join("
")} + end + sorter.each_with_index do |hash,idx| + k = hash.keys[0] + v = hash.values[0] + string += (idx == 0 ? "" : "")+(k =~ /lazar/i ? "" end - haml :prediction + string += "
" : "") + # keyword + string += "#{k}:" + string += "" + # values + string += "#{v}" + string += "
" + return JSON.pretty_generate(:prediction => [html]) end end -get "/report/:id/?" do - prediction_model = Model::Validation.find params[:id] - bad_request_error "model with id: '#{params[:id]}' not found." unless prediction_model - report = qmrf_report params[:id] - # output - t = Tempfile.new - t << report.to_xml - name = prediction_model.species.sub(/\s/,"-")+"-"+prediction_model.endpoint.downcase.sub(/\s/,"-") - send_file t.path, :filename => "QMRF_report_#{name.gsub!(/[^0-9A-Za-z]/, '_')}.xml", :type => "application/xml", :disposition => "attachment" +# get individual compound details +get '/prediction/:neighbor/details/?' do + @compound = OpenTox::Compound.find params[:neighbor] + @smiles = @compound.smiles + begin + @names = @compound.names.nil? ? "No names for this compound available." : @compound.names + rescue + @names = "No names for this compound available." + end + @inchi = @compound.inchi.gsub("InChI=", "") + + haml :details, :layout => false end get '/license' do @@ -314,7 +469,7 @@ get '/license' do haml :license, :layout => false end -get '/faq' do +get '/predict/faq' do @faq = RDiscount.new(File.read("FAQ.md")).to_html haml :faq#, :layout => false end @@ -324,6 +479,7 @@ get '/style.css' do scss :style end +# for swagger representation get '/swagger-ui.css' do headers 'Content-Type' => 'text/css; charset=utf-8' scss :style -- cgit v1.2.3 From 993102a8b2f131526e4578f514b4392c77f2b47d Mon Sep 17 00:00:00 2001 From: gebele Date: Mon, 3 Dec 2018 15:18:06 +0000 Subject: fixed csv download;added analytics and batch files --- application.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index dd62fb6..aaa18ae 100644 --- a/application.rb +++ b/application.rb @@ -123,7 +123,10 @@ get '/predict/dataset/:name' do response['Content-Type'] = "text/csv" dataset = Dataset.find_by(:name=>params[:name]) csv = dataset.to_csv - csv + t = Tempfile.new + t << csv + name = params[:name] + ".csv" + send_file t.path, :filename => name, :type => "text/csv", :disposition => "attachment" end get '/predict/:tmppath/:filename/?' do -- cgit v1.2.3 From 5a3be4190688bc8240327930b3e953b09ecc9d9e Mon Sep 17 00:00:00 2001 From: gebele Date: Tue, 28 May 2019 14:25:52 +0000 Subject: before clean up --- application.rb | 382 ++++++++++++++------------------------------------------- 1 file changed, 93 insertions(+), 289 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index aaa18ae..508a8c6 100644 --- a/application.rb +++ b/application.rb @@ -1,11 +1,9 @@ require 'rdiscount' require_relative 'qmrf_report.rb' require_relative 'task.rb' -require_relative 'prediction.rb' -require_relative 'batch.rb' require_relative 'helper.rb' include OpenTox - +=begin [ "api.rb", "compound.rb", @@ -18,7 +16,7 @@ include OpenTox "swagger.rb", "validation.rb" ].each{ |f| require_relative "./lib/#{f}" } - +=end configure :production, :development do STDOUT.sync = true @@ -26,8 +24,8 @@ configure :production, :development do $logger.level = Logger::DEBUG enable :reloader also_reload './helper.rb' - also_reload './prediction.rb' - also_reload './batch.rb' + also_reload './qmrf_report.rb' +=begin [ "api.rb", "compound.rb", @@ -40,8 +38,9 @@ configure :production, :development do "swagger.rb", "validation.rb" ].each{ |f| also_reload "./lib/#{f}" } +=end end - +=begin before do $paths = [ "/", @@ -62,11 +61,15 @@ before do @version = File.read("VERSION").chomp end end +=end +before do + @version = File.read("VERSION").chomp +end not_found do redirect to('/predict') end - +=begin error do if request.path == "/" || $paths.include?(request.path.split("/")[1]) @accept = request.env['HTTP_ACCEPT'] @@ -77,33 +80,40 @@ error do haml :error end end +=end +error do + @error = request.env['sinatra.error'] + haml :error +end +=begin # https://github.com/britg/sinatra-cross_origin#responding-to-options options "*" do response.headers["Allow"] = "HEAD,GET,PUT,POST,DELETE,OPTIONS" response.headers["Access-Control-Allow-Headers"] = "X-Requested-With, X-HTTP-Method-Override, Content-Type, Cache-Control, Accept" 200 end - +=end get '/predict/?' do @models = OpenTox::Model::Validation.all - @models = @models.delete_if{|m| m.model.name =~ /\b(Net cell association)\b/} @endpoints = @models.collect{|m| m.endpoint}.sort.uniq - if @models.count > 0 - rodent_index = 0 - @models.each_with_index{|model,idx| rodent_index = idx if model.species =~ /Rodent/} - @models.insert(rodent_index-1,@models.delete_at(rodent_index)) - end @models.count > 0 ? (haml :predict) : (haml :info) end get '/predict/modeldetails/:model' do model = OpenTox::Model::Validation.find params[:model] + training_dataset = model.model.training_dataset + data_entries = training_dataset.data_entries crossvalidations = OpenTox::Validation::RepeatedCrossValidation.find(model.repeated_crossvalidation_id).crossvalidations - return haml :model_details, :layout=> false, :locals => {:model => model, :crossvalidations => crossvalidations} + return haml :model_details, :layout=> false, :locals => {:model => model, + :crossvalidations => crossvalidations, + :training_dataset => training_dataset, + :data_entries => data_entries + } end +#TODO fix update get "/predict/report/:id/?" do prediction_model = Model::Validation.find params[:id] bad_request_error "model with id: '#{params[:id]}' not found." unless prediction_model @@ -119,6 +129,7 @@ get '/predict/jme_help/?' do File.read(File.join('views','jme_help.html')) end +# download training dataset get '/predict/dataset/:name' do response['Content-Type'] = "text/csv" dataset = Dataset.find_by(:name=>params[:name]) @@ -129,226 +140,54 @@ get '/predict/dataset/:name' do send_file t.path, :filename => name, :type => "text/csv", :disposition => "attachment" end -get '/predict/:tmppath/:filename/?' do - response['Content-Type'] = "text/csv" - path = "/tmp/#{params[:tmppath]}" - send_file path, :filename => "lazar_batch_prediction_#{params[:filename]}", :type => "text/csv", :disposition => "attachment" -end - -get '/predict/csv/:task/:model/:filename/?' do - response['Content-Type'] = "text/csv" - filename = params[:filename] =~ /\.csv$/ ? params[:filename].gsub(/\.csv$/,"") : params[:filename] - task = Task.find params[:task].to_s - m = Model::Validation.find params[:model].to_s - dataset = Batch.find_by(:name => filename) - @ids = dataset.ids - warnings = dataset.warnings.blank? ? nil : dataset.warnings.join("\n") - unless warnings.nil? - @parse = [] - warnings.split("\n").each do |warning| - if warning =~ /^Cannot/ - smi = warning.split("SMILES compound").last.split("at").first - line = warning.split("SMILES compound").last.split("at line").last.split("of").first.strip.to_i - @parse << "Cannot parse SMILES compound#{smi}at line #{line} of #{dataset.source.split("/").last}\n" - end - end - keys_array = [] - warnings.split("\n").each do |warning| - if warning =~ /^Duplicate/ - text = warning.split("ID").first - numbers = warning.split("ID").last.split("and") - keys_array << numbers.collect{|n| n.strip.to_i} - end - end - @dups = {} - keys_array.each do |keys| - keys.each do |key| - @dups[key] = "Duplicate compound at ID #{keys.join(" and ")}\n" - end - end - end - endpoint = "#{m.endpoint}_(#{m.species})" +# download batch predicton file +get '/predict/batch/download/?' do + task = Task.find params[:tid] + prediction_dataset = Dataset.find task.dataset_id + filename = prediction_dataset.name tempfile = Tempfile.new - header = task.csv - lines = [] - task.predictions[params[:model]].each_with_index do |hash,idx| - identifier = hash.keys[0] - prediction_id = hash.values[0] - # add duplicate warning at the end of a line if ID matches - if @dups && @dups[idx+1] - if prediction_id.is_a? BSON::ObjectId - if @ids.blank? - lines << "#{idx+1},#{identifier},#{Prediction.find(prediction_id).csv.tr("\n","")},#{@dups[idx+1]}" - else - lines << "#{idx+1},#{@ids[idx]},#{identifier},#{Prediction.find(prediction_id).csv.tr("\n","")},#{@dups[idx+1]}" - end - end - else - if prediction_id.is_a? BSON::ObjectId - if @ids.blank? - lines << "#{idx+1},#{identifier},#{Prediction.find(prediction_id).csv}" - else - lines << "#{idx+1},#{@ids[idx]},#{identifier},#{Prediction.find(prediction_id).csv}" - end - else - if @ids.blank? - lines << "#{idx+1},#{identifier},#{p}\n" - else - lines << "#{idx+1},#{@ids[idx]}#{identifier},#{p}\n" - end - end - end - end - (@parse && !@parse.blank?) ? tempfile.write(header+lines.join("")+"\n"+@parse.join("\n")) : tempfile.write(header+lines.join("")) - #tempfile.write(header+lines.join("")) + tempfile << prediction_dataset.to_csv tempfile.rewind - send_file tempfile, :filename => "#{Time.now.strftime("%Y-%m-%d")}_lazar_batch_prediction_#{endpoint}_#{filename}.csv", :type => "text/csv", :disposition => "attachment" + response['Content-Type'] = "text/csv" + send_file tempfile, :filename => "#{Time.now.strftime("%Y-%m-%d")}_lazar_batch_prediction_#{filename}.csv", :type => "text/csv", :disposition => "attachment" end post '/predict/?' do # process batch prediction if !params[:fileselect].blank? - next if params[:fileselect][:filename] !~ /\.csv$/ bad_request_error "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file." end @filename = params[:fileselect][:filename] - begin - File.open('tmp/' + params[:fileselect][:filename], "w") do |f| - f.write(params[:fileselect][:tempfile].read) - end - input = Batch.from_csv_file File.join("tmp", params[:fileselect][:filename]) - $logger.debug "Processing '#{params[:fileselect][:filename]}'" - if input.class == OpenTox::Batch - @dataset = input - @compounds = @dataset.compounds - @identifiers = @dataset.identifiers - @ids = @dataset.ids - else - File.delete File.join("tmp", params[:fileselect][:filename]) - bad_request_error "Could not serialize file '#{@filename}'." - end - rescue - File.delete File.join("tmp", params[:fileselect][:filename]) - bad_request_error "Could not serialize file '#{@filename}'." + File.open('tmp/' + params[:fileselect][:filename], "w") do |f| + f.write(params[:fileselect][:tempfile].read) end - - if @compounds.size == 0 - message = @dataset.warnings - @dataset.delete - bad_request_error message - end - + input = Dataset.from_csv_file File.join("tmp", params[:fileselect][:filename]) + $logger.debug "Processing '#{params[:fileselect][:filename]}'" + @compounds_size = input.compounds.size @models = params[:selection].keys - # for single predictions in batch @tasks = [] @models.each{|m| t = Task.new; t.save; @tasks << t} @predictions = {} - task = Task.run do - @models.each_with_index do |model,idx| - t = @tasks[idx] - m = Model::Validation.find model - type = (m.regression? ? "Regression" : "Classification") - # add header for regression - if type == "Regression" - unit = (type == "Regression") ? "(#{m.unit})" : "" - converted_unit = (type == "Regression") ? "#{m.unit =~ /\b(mmol\/L)\b/ ? "(mg/L)" : "(mg/kg_bw/day)"}" : "" - if @ids.blank? - header = "ID,Input,Endpoint,Unique SMILES,inTrainingSet,Measurements #{unit},Prediction #{unit},Prediction #{converted_unit},"\ - "Prediction Interval Low #{unit},Prediction Interval High #{unit},"\ - "Prediction Interval Low #{converted_unit},Prediction Interval High #{converted_unit},"\ - "inApplicabilityDomain,Note\n" - else - header = "ID,Original ID,Input,Endpoint,Unique SMILES,inTrainingSet,Measurements #{unit},Prediction #{unit},Prediction #{converted_unit},"\ - "Prediction Interval Low #{unit},Prediction Interval High #{unit},"\ - "Prediction Interval Low #{converted_unit},Prediction Interval High #{converted_unit},"\ - "inApplicabilityDomain,Note\n" - end - end - # add header for classification - if type == "Classification" - av = m.prediction_feature.accept_values - if @ids.blank? - header = "ID,Input,Endpoint,Unique SMILES,inTrainingSet,Measurements,Prediction,"\ - "predProbability #{av[0]},predProbability #{av[1]},inApplicabilityDomain,Note\n" - else - header = "ID,Original ID,Input,Endpoint,Unique SMILES,inTrainingSet,Measurements,Prediction,"\ - "predProbability #{av[0]},predProbability #{av[1]},inApplicabilityDomain,Note\n" - end - end - # predict compounds - p = 100.0/@compounds.size - counter = 1 - predictions = [] - @compounds.each_with_index do |cid,idx| - compound = Compound.find cid - if Prediction.where(compound: compound.id, model: m.id).exists? - prediction_object = Prediction.find_by(compound: compound.id, model: m.id) - prediction = prediction_object.prediction - prediction_id = prediction_object.id - # in case prediction object was created by single prediction - if prediction_object.csv.blank? - prediction_object[:csv] = prediction_to_csv(m,compound,prediction) - prediction_object.save - end - # identifier - identifier = @identifiers[idx] - else - prediction = m.predict(compound) - # save prediction object - prediction_object = Prediction.new - prediction_id = prediction_object.id - prediction_object[:compound] = compound.id - prediction_object[:model] = m.id - # add additionally fields for html representation - unless prediction[:value].blank? || type == "Classification" - prediction[:prediction_value] = "#{prediction[:value].delog10.signif(3)} #{unit}" - prediction["converted_prediction_value"] = "#{compound.mmol_to_mg(prediction[:value].delog10).signif(3)} #{converted_unit}" - end - unless prediction[:prediction_interval].blank? - interval = prediction[:prediction_interval] - prediction[:interval] = "#{interval[1].delog10.signif(3)} - #{interval[0].delog10.signif(3)} #{unit}" - prediction[:converted_interval] = "#{compound.mmol_to_mg(interval[1].delog10).signif(3)} - #{compound.mmol_to_mg(interval[0].delog10).signif(3)} #{converted_unit}" - end - prediction["unit"] = unit - prediction["converted_unit"] = converted_unit - if prediction[:measurements].is_a?(Array) - prediction["measurements_string"] = (type == "Regression") ? prediction[:measurements].collect{|value| "#{value.delog10.signif(3)} #{unit}"} : prediction[:measurements].join("
") - prediction["converted_measurements"] = prediction[:measurements].collect{|value| "#{compound.mmol_to_mg(value.delog10).signif(3)} #{unit =~ /mmol\/L/ ? "(mg/L)" : "(mg/kg_bw/day)"}"} if type == "Regression" - else - output["measurements_string"] = (type == "Regression") ? "#{prediction[:measurements].delog10.signif(3)} #{unit}}" : prediction[:measurements] - output["converted_measurements"] = "#{compound.mmol_to_mg(prediction[:measurements].delog10).signif(3)} #{(unit =~ /\b(mmol\/L)\b/) ? "(mg/L)" : "(mg/kg_bw/day)"}" if type == "Regression" - end - # store in prediction_object - prediction_object[:prediction] = prediction - prediction_object[:csv] = prediction_to_csv(m,compound,prediction) - prediction_object.save - - # identifier - identifier = @identifiers[idx] - end - # collect prediction_object ids with identifier - predictions << {identifier => prediction_id} - t.update_percent((counter*p).ceil > 100 ? 100 : (counter*p).ceil) - counter += 1 - end - # write csv - t[:csv] = header - # write predictions - @predictions["#{model}"] = predictions - # save task - # append predictions as last action otherwise they won't save - # mongoid works with shallow copy via #dup - t[:predictions] = @predictions + maintask = Task.run do + @models.each_with_index do |model_id,idx| + t = @tasks[idx] + t.update_percent(1) + prediction = {} + model = Model::Validation.find model_id + t.update_percent(10) + prediction_dataset = model.predict input + t[:dataset_id] = prediction_dataset.id + t.update_percent(90) + prediction[model_id] = prediction_dataset.id.to_s + t[:predictions] = prediction + t[:csv] = prediction_dataset.to_csv + t.update_percent(100) t.save - end#models - - end#main task - @pid = task.pid - - #@dataset.delete - #File.delete File.join("tmp", params[:fileselect][:filename]) + end + end + @pid = maintask.pid return haml :batch else # single compound prediction @@ -357,30 +196,20 @@ post '/predict/?' do @identifier = params[:identifier].strip $logger.debug "input:#{@identifier}" # get compound from SMILES - @compound = Compound.from_smiles @identifier - bad_request_error "'#{@identifier}' is not a valid SMILES string." if @compound.blank? - + begin + @compound = Compound.from_smiles @identifier + rescue + @error = "'#{@identifier}' is not a valid SMILES string." unless @compound + return haml :error + end @models = [] @predictions = [] - @toxtree = false params[:selection].keys.each do |model_id| model = Model::Validation.find model_id @models << model - if Prediction.where(compound: @compound.id, model: model.id).exists? - prediction_object = Prediction.find_by(compound: @compound.id, model: model.id) - prediction = prediction_object.prediction - @predictions << prediction - else - prediction_object = Prediction.new - prediction = model.predict(@compound) - prediction_object[:compound] = @compound.id - prediction_object[:model] = model.id - prediction_object[:prediction] = prediction - prediction_object.save - @predictions << prediction - end + prediction = model.predict(@compound) + @predictions << prediction end - haml :prediction end end @@ -394,62 +223,33 @@ get '/prediction/task/?' do task = Task.find(params[:predictions]) pageSize = params[:pageSize].to_i - 1 pageNumber= params[:pageNumber].to_i - 1 - predictions = task.predictions[params[:model]].collect{|hash| hash.values[0]} - prediction_object = Prediction.find predictions[pageNumber] - prediction = prediction_object.prediction - compound = Compound.find prediction_object.compound - model = Model::Validation.find prediction_object.model - image = compound.svg - smiles = compound.smiles - type = (model.regression? ? "Regression" : "Classification") - html = "" - html += "" - string = "" string += "
#{image}
#{smiles}
" - sorter = [] - if prediction[:info] - prediction[:info] = "This compound was part of the training dataset. All information from this compound was "\ - "removed from the training data before the prediction to obtain unbiased results." - sorter << {"Info" => prediction[:info]} - if prediction["measurements_string"].kind_of?(Array) - sorter << {"Measured activity" => "#{prediction["measurements_string"].join(";")}
#{prediction["converted_measurements"].join(";")}"} + csv = CSV.parse(task.csv) + header = csv.shift + string = "" - html += "#{string}
" + # find canonical smiles column + cansmi = 0 + header.each_with_index do |h,idx| + cansmi = idx if h =~ /Canonical SMILES/ + string += "" + end + string += "" + string += "" + csv[pageNumber].each_with_index do |line,idx| + if idx == cansmi + c = Compound.from_smiles line + string += "" else - sorter << {"Measured activity" => "#{prediction["measurements_string"]}
#{prediction["converted_measurements"]}"} + string += "" end end - - # regression - if prediction[:value] && type == "Regression" - sorter << {"Prediction" => "#{prediction["prediction_value"]}
#{prediction["converted_prediction_value"]}"} - sorter << {"95% Prediction interval" => "#{prediction[:interval]}
#{prediction["converted_interval"]}"} - sorter << {"Warnings" => prediction[:warnings].join("
")} - elsif !prediction[:value] && type == "Regression" - sorter << {"Prediction" => ""} - sorter << {"95% Prediction interval" => ""} - sorter << {"Warnings" => prediction[:warnings].join("
")} - # classification - elsif prediction[:value] && type == "Classification" - sorter << {"Prediction" => prediction[:value]} - sorter << {"Probability" => prediction[:probabilities].collect{|k,v| "#{k}: #{v.signif(3)}"}.join("
")} - elsif !prediction[:value] && type == "Classification" - sorter << {"Prediction" => ""} - sorter << {"Probability" => ""} - #else - sorter << {"Warnings" => prediction[:warnings].join("
")} - end - sorter.each_with_index do |hash,idx| - k = hash.keys[0] - v = hash.values[0] - string += (idx == 0 ? "" : "")+(k =~ /lazar/i ? "" - end + string += "" string += "
#{h}
#{line}
" \ + "" \ + "#{embedded_svg(c.svg, title: "click for details")}" \ + "		#{line.numeric? && line.include?(".") ? line.to_f.signif(3) : line}
" : "") - # keyword - string += "#{k}:" - string += "" - # values - string += "#{v}" - string += "
" - return JSON.pretty_generate(:prediction => [html]) + return JSON.pretty_generate(:prediction => [string]) end end @@ -462,7 +262,7 @@ get '/prediction/:neighbor/details/?' do rescue @names = "No names for this compound available." end - @inchi = @compound.inchi.gsub("InChI=", "") + @inchi = @compound.inchi haml :details, :layout => false end @@ -477,6 +277,10 @@ get '/predict/faq' do haml :faq#, :layout => false end +get '/help' do + haml :help +end + get '/style.css' do headers 'Content-Type' => 'text/css; charset=utf-8' scss :style -- cgit v1.2.3 From f37f0b654b36b66c133755c01f033859c35197f6 Mon Sep 17 00:00:00 2001 From: gebele Date: Wed, 29 May 2019 11:12:33 +0000 Subject: removed comment --- application.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 508a8c6..43030d0 100644 --- a/application.rb +++ b/application.rb @@ -113,7 +113,6 @@ get '/predict/modeldetails/:model' do } end -#TODO fix update get "/predict/report/:id/?" do prediction_model = Model::Validation.find params[:id] bad_request_error "model with id: '#{params[:id]}' not found." unless prediction_model -- cgit v1.2.3 From 741701df8ff0861b3607a30e9aaf8b8a0c303cdf Mon Sep 17 00:00:00 2001 From: gebele Date: Thu, 13 Jun 2019 15:28:59 +0000 Subject: update with API --- application.rb | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 43030d0..73ec11a 100644 --- a/application.rb +++ b/application.rb @@ -3,7 +3,7 @@ require_relative 'qmrf_report.rb' require_relative 'task.rb' require_relative 'helper.rb' include OpenTox -=begin + [ "api.rb", "compound.rb", @@ -16,7 +16,6 @@ include OpenTox "swagger.rb", "validation.rb" ].each{ |f| require_relative "./lib/#{f}" } -=end configure :production, :development do STDOUT.sync = true @@ -25,7 +24,6 @@ configure :production, :development do enable :reloader also_reload './helper.rb' also_reload './qmrf_report.rb' -=begin [ "api.rb", "compound.rb", @@ -38,12 +36,10 @@ configure :production, :development do "swagger.rb", "validation.rb" ].each{ |f| also_reload "./lib/#{f}" } -=end end -=begin + before do $paths = [ - "/", "api", "compound", "dataset", @@ -54,24 +50,20 @@ before do "substance", "swagger", "validation"] - if request.path == "/" || $paths.include?(request.path.split("/")[1]) - @accept = request.env['HTTP_ACCEPT'] + if request.path.split("/")[1] == "api" || $paths.include?(request.path.split("/")[2]) + @accept = request.env['HTTP_ACCEPT'].split(",").first response['Content-Type'] = @accept else @version = File.read("VERSION").chomp end end -=end -before do - @version = File.read("VERSION").chomp -end not_found do redirect to('/predict') end -=begin + error do - if request.path == "/" || $paths.include?(request.path.split("/")[1]) + if request.path.split("/")[1] == "api" || $paths.include?(request.path.split("/")[2]) @accept = request.env['HTTP_ACCEPT'] response['Content-Type'] = @accept @accept == "text/plain" ? request.env['sinatra.error'] : request.env['sinatra.error'].to_json @@ -80,20 +72,14 @@ error do haml :error end end -=end -error do - @error = request.env['sinatra.error'] - haml :error -end -=begin # https://github.com/britg/sinatra-cross_origin#responding-to-options options "*" do response.headers["Allow"] = "HEAD,GET,PUT,POST,DELETE,OPTIONS" response.headers["Access-Control-Allow-Headers"] = "X-Requested-With, X-HTTP-Method-Override, Content-Type, Cache-Control, Accept" 200 end -=end + get '/predict/?' do @models = OpenTox::Model::Validation.all @endpoints = @models.collect{|m| m.endpoint}.sort.uniq -- cgit v1.2.3 From be9d7e0dd360328d3ef7db77128527c40819cc1c Mon Sep 17 00:00:00 2001 From: gebele Date: Mon, 17 Jun 2019 10:21:24 +0000 Subject: add PubChem links for compound cid --- application.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 73ec11a..a2ec091 100644 --- a/application.rb +++ b/application.rb @@ -3,6 +3,7 @@ require_relative 'qmrf_report.rb' require_relative 'task.rb' require_relative 'helper.rb' include OpenTox +PUBCHEM_CID_URI = PUBCHEM_URI.split("/")[0..-3].join("/")+"/compound/" [ "api.rb", -- cgit v1.2.3 From 3869670b3acfb4de982f45ea24af940eccac5474 Mon Sep 17 00:00:00 2001 From: gebele Date: Tue, 18 Jun 2019 12:49:35 +0000 Subject: update routes and mime type and generate server uri for API file --- application.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index a2ec091..87d676b 100644 --- a/application.rb +++ b/application.rb @@ -51,11 +51,14 @@ before do "substance", "swagger", "validation"] - if request.path.split("/")[1] == "api" || $paths.include?(request.path.split("/")[2]) + if request.path =~ /predict/ @accept = request.env['HTTP_ACCEPT'].split(",").first response['Content-Type'] = @accept - else + halt 400, "Mime type #{@accept} is not supported." unless @accept == "text/html" || "*/*" @version = File.read("VERSION").chomp + else + @accept = request.env['HTTP_ACCEPT'].split(",").first + response['Content-Type'] = @accept end end @@ -253,7 +256,7 @@ get '/prediction/:neighbor/details/?' do haml :details, :layout => false end -get '/license' do +get '/predict/license' do @license = RDiscount.new(File.read("LICENSE.md")).to_html haml :license, :layout => false end @@ -263,7 +266,7 @@ get '/predict/faq' do haml :faq#, :layout => false end -get '/help' do +get '/predict/help' do haml :help end @@ -273,7 +276,7 @@ get '/style.css' do end # for swagger representation -get '/swagger-ui.css' do +get '/api/swagger-ui.css' do headers 'Content-Type' => 'text/css; charset=utf-8' scss :style end -- cgit v1.2.3 From bba7061d7ff2420f4eb2b6f88362edb71bc6bf25 Mon Sep 17 00:00:00 2001 From: gebele Date: Tue, 18 Jun 2019 15:04:55 +0000 Subject: return training dataset from source --- application.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 87d676b..edbd289 100644 --- a/application.rb +++ b/application.rb @@ -94,7 +94,7 @@ get '/predict/modeldetails/:model' do model = OpenTox::Model::Validation.find params[:model] training_dataset = model.model.training_dataset data_entries = training_dataset.data_entries - crossvalidations = OpenTox::Validation::RepeatedCrossValidation.find(model.repeated_crossvalidation_id).crossvalidations + crossvalidations = model.crossvalidations return haml :model_details, :layout=> false, :locals => {:model => model, :crossvalidations => crossvalidations, @@ -122,7 +122,7 @@ end get '/predict/dataset/:name' do response['Content-Type'] = "text/csv" dataset = Dataset.find_by(:name=>params[:name]) - csv = dataset.to_csv + csv = File.read dataset.source t = Tempfile.new t << csv name = params[:name] + ".csv" -- cgit v1.2.3 From 2d4782c4e8df908efa0973ee842593ead58d5980 Mon Sep 17 00:00:00 2001 From: gebele Date: Thu, 27 Jun 2019 08:37:33 +0000 Subject: use prediction to csv;edit info text --- application.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index edbd289..060d163 100644 --- a/application.rb +++ b/application.rb @@ -135,7 +135,7 @@ get '/predict/batch/download/?' do prediction_dataset = Dataset.find task.dataset_id filename = prediction_dataset.name tempfile = Tempfile.new - tempfile << prediction_dataset.to_csv + tempfile << prediction_dataset.to_prediction_csv tempfile.rewind response['Content-Type'] = "text/csv" send_file tempfile, :filename => "#{Time.now.strftime("%Y-%m-%d")}_lazar_batch_prediction_#{filename}.csv", :type => "text/csv", :disposition => "attachment" @@ -171,7 +171,7 @@ post '/predict/?' do t.update_percent(90) prediction[model_id] = prediction_dataset.id.to_s t[:predictions] = prediction - t[:csv] = prediction_dataset.to_csv + t[:csv] = prediction_dataset.to_prediction_csv t.update_percent(100) t.save end -- cgit v1.2.3 From 86a45822c29cef7fb84871416427f96782e4ecf5 Mon Sep 17 00:00:00 2001 From: gebele Date: Fri, 5 Jul 2019 08:26:18 +0000 Subject: reorder validation statistics --- application.rb | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 060d163..db4c230 100644 --- a/application.rb +++ b/application.rb @@ -120,25 +120,26 @@ end # download training dataset get '/predict/dataset/:name' do - response['Content-Type'] = "text/csv" dataset = Dataset.find_by(:name=>params[:name]) csv = File.read dataset.source + name = params[:name] + ".csv" t = Tempfile.new t << csv - name = params[:name] + ".csv" + t.rewind + response['Content-Type'] = "text/csv" send_file t.path, :filename => name, :type => "text/csv", :disposition => "attachment" end # download batch predicton file get '/predict/batch/download/?' do task = Task.find params[:tid] - prediction_dataset = Dataset.find task.dataset_id - filename = prediction_dataset.name - tempfile = Tempfile.new - tempfile << prediction_dataset.to_prediction_csv - tempfile.rewind + dataset = Dataset.find task.dataset_id + name = dataset.name + ".csv" + t = Tempfile.new + t << dataset.to_prediction_csv + t.rewind response['Content-Type'] = "text/csv" - send_file tempfile, :filename => "#{Time.now.strftime("%Y-%m-%d")}_lazar_batch_prediction_#{filename}.csv", :type => "text/csv", :disposition => "attachment" + send_file t.path, :filename => "#{Time.now.strftime("%Y-%m-%d")}_lazar_batch_prediction_#{name}", :type => "text/csv", :disposition => "attachment" end post '/predict/?' do -- cgit v1.2.3 From 2b48d7a57b340b2ffb344774529ed84430883035 Mon Sep 17 00:00:00 2001 From: gebele Date: Wed, 10 Jul 2019 09:01:52 +0000 Subject: ensure correct response content-type --- application.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index db4c230..7cebb2b 100644 --- a/application.rb +++ b/application.rb @@ -168,10 +168,13 @@ post '/predict/?' do model = Model::Validation.find model_id t.update_percent(10) prediction_dataset = model.predict input + t.update_percent(70) t[:dataset_id] = prediction_dataset.id - t.update_percent(90) + t.update_percent(75) prediction[model_id] = prediction_dataset.id.to_s + t.update_percent(80) t[:predictions] = prediction + t.update_percent(90) t[:csv] = prediction_dataset.to_prediction_csv t.update_percent(100) t.save @@ -208,6 +211,7 @@ end get '/prediction/task/?' do if params[:turi] task = Task.find(params[:turi].to_s) + response['Content-Type'] = "application/json" return JSON.pretty_generate(:percent => task.percent) elsif params[:predictions] task = Task.find(params[:predictions]) @@ -239,6 +243,7 @@ get '/prediction/task/?' do end string += "
" + response['Content-Type'] = "application/json" return JSON.pretty_generate(:prediction => [string]) end end -- cgit v1.2.3 From 5efd8128dc5bdb9ffc0aa4798f8aa303973794e6 Mon Sep 17 00:00:00 2001 From: gebele Date: Thu, 11 Jul 2019 08:20:10 +0000 Subject: moved form js functions to lazar-gui.js --- application.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 7cebb2b..4ed4a31 100644 --- a/application.rb +++ b/application.rb @@ -96,6 +96,7 @@ get '/predict/modeldetails/:model' do data_entries = training_dataset.data_entries crossvalidations = model.crossvalidations + response['Content-Type'] = "text/html" return haml :model_details, :layout=> false, :locals => {:model => model, :crossvalidations => crossvalidations, :training_dataset => training_dataset, -- cgit v1.2.3 From 5490f6214f3a22216f7980c7d46f6d9fe10a4924 Mon Sep 17 00:00:00 2001 From: gebele Date: Thu, 11 Jul 2019 15:59:13 +0000 Subject: simplify js code;reorder js code; --- application.rb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 4ed4a31..76137b9 100644 --- a/application.rb +++ b/application.rb @@ -85,6 +85,13 @@ options "*" do end get '/predict/?' do + if params[:tpid] + begin + Process.kill(9,params[:tpid].to_i) if !params[:tpid].blank? + rescue + nil + end + end @models = OpenTox::Model::Validation.all @endpoints = @models.collect{|m| m.endpoint}.sort.uniq @models.count > 0 ? (haml :predict) : (haml :info) -- cgit v1.2.3 From 592fa64120e2712c46dd020fbda0c128c69467b7 Mon Sep 17 00:00:00 2001 From: gebele Date: Fri, 12 Jul 2019 10:50:51 +0000 Subject: kill task pid if page is reloaded --- application.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 76137b9..0d5da57 100644 --- a/application.rb +++ b/application.rb @@ -221,6 +221,14 @@ get '/prediction/task/?' do task = Task.find(params[:turi].to_s) response['Content-Type'] = "application/json" return JSON.pretty_generate(:percent => task.percent) + elsif params[:ktpid] + begin + Process.kill(9,params[:ktpid].to_i) if !params[:ktpid].blank? + rescue + nil + end + response['Content-Type'] = "application/json" + return JSON.pretty_generate(:ktpid => params[:ktpid]) elsif params[:predictions] task = Task.find(params[:predictions]) pageSize = params[:pageSize].to_i - 1 -- cgit v1.2.3 From e0985405e2fdb0fe8c4740ccb256d3ac2d1fd5be Mon Sep 17 00:00:00 2001 From: gebele Date: Tue, 16 Jul 2019 14:20:09 +0000 Subject: use png format for plots avoiding css overwrite from svg objects and better browser performance --- application.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 0d5da57..f5f3486 100644 --- a/application.rb +++ b/application.rb @@ -102,6 +102,19 @@ get '/predict/modeldetails/:model' do training_dataset = model.model.training_dataset data_entries = training_dataset.data_entries crossvalidations = model.crossvalidations + if model.classification? + crossvalidations.each do |cv| + File.open(File.join('public', "#{cv.id}.png"), 'w') do |file| + file.write(cv.probability_plot(format: "png")) + end unless File.exists? File.join('public', "#{cv.id}.png") + end + else + crossvalidations.each do |cv| + File.open(File.join('public', "#{cv.id}.png"), 'w') do |file| + file.write(cv.correlation_plot(format: "png")) + end unless File.exists? File.join('public', "#{cv.id}.png") + end + end response['Content-Type'] = "text/html" return haml :model_details, :layout=> false, :locals => {:model => model, -- cgit v1.2.3 From 807b87cf18b42c055e62c91e31b05a728a7fdd42 Mon Sep 17 00:00:00 2001 From: gebele Date: Mon, 29 Jul 2019 14:53:44 +0000 Subject: optimize batch table size; mute task progress js functions --- application.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index f5f3486..fa2a728 100644 --- a/application.rb +++ b/application.rb @@ -157,7 +157,9 @@ get '/predict/batch/download/?' do dataset = Dataset.find task.dataset_id name = dataset.name + ".csv" t = Tempfile.new - t << dataset.to_prediction_csv + # to_prediction_csv takes too much time; use task.csv instead which is the same + #t << dataset.to_prediction_csv + t << task.csv t.rewind response['Content-Type'] = "text/csv" send_file t.path, :filename => "#{Time.now.strftime("%Y-%m-%d")}_lazar_batch_prediction_#{name}", :type => "text/csv", :disposition => "attachment" @@ -253,21 +255,21 @@ get '/prediction/task/?' do cansmi = 0 header.each_with_index do |h,idx| cansmi = idx if h =~ /Canonical SMILES/ - string += "#{h}" + string += "#{h}" end string += "" string += "" csv[pageNumber].each_with_index do |line,idx| if idx == cansmi c = Compound.from_smiles line - string += "#{line}
" \ + string += "#{line}
" \ "" \ "#{embedded_svg(c.svg, title: "click for details")}" \ "" else - string += "#{line.numeric? && line.include?(".") ? line.to_f.signif(3) : line}" + string += "#{line.numeric? && line.include?(".") ? line.to_f.signif(3) : (line.nil? ? line : line.gsub(" ","
"))}" end end string += "" -- cgit v1.2.3 From 1ffdd765fe4b4d9d80626070dea652fe467a2fa5 Mon Sep 17 00:00:00 2001 From: gebele Date: Thu, 1 Aug 2019 16:48:14 +0000 Subject: introduce task for upload and parse to dataset --- application.rb | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index fa2a728..8872e5f 100644 --- a/application.rb +++ b/application.rb @@ -167,7 +167,7 @@ end post '/predict/?' do # process batch prediction - if !params[:fileselect].blank? + unless params[:fileselect].blank? if params[:fileselect][:filename] !~ /\.csv$/ bad_request_error "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file." end @@ -175,9 +175,20 @@ post '/predict/?' do File.open('tmp/' + params[:fileselect][:filename], "w") do |f| f.write(params[:fileselect][:tempfile].read) end - input = Dataset.from_csv_file File.join("tmp", params[:fileselect][:filename]) - $logger.debug "Processing '#{params[:fileselect][:filename]}'" - @compounds_size = input.compounds.size + uploadTask = Task.new + uploadTask.save + uploadDataset = Task.run do + t = uploadTask + t.update_percent(1) + $logger.debug "Processing '#{params[:fileselect][:filename]}'" + @input = Dataset.from_csv_file File.join("tmp", params[:fileselect][:filename]) + t.update_percent(100) + t.save + end + @upid = uploadTask.id + + #TODO route for compound size + @compounds_size = 0 #@input.compounds.size @models = params[:selection].keys @tasks = [] @models.each{|m| t = Task.new; t.save; @tasks << t} @@ -190,6 +201,10 @@ post '/predict/?' do prediction = {} model = Model::Validation.find model_id t.update_percent(10) + input = Dataset.find_by(:source => "tmp/"+@filename) + until input + sleep 1 + end prediction_dataset = model.predict input t.update_percent(70) t[:dataset_id] = prediction_dataset.id @@ -232,10 +247,18 @@ post '/predict/?' do end get '/prediction/task/?' do + # returns task progress in percentage if params[:turi] task = Task.find(params[:turi].to_s) response['Content-Type'] = "application/json" - return JSON.pretty_generate(:percent => task.percent) + if task.dataset_id + d = Dataset.find task.dataset_id + size = d.compounds.size + return JSON.pretty_generate(:percent => task.percent, :size => size) + else + return JSON.pretty_generate(:percent => task.percent) + end + # kills task process id elsif params[:ktpid] begin Process.kill(9,params[:ktpid].to_i) if !params[:ktpid].blank? @@ -244,6 +267,7 @@ get '/prediction/task/?' do end response['Content-Type'] = "application/json" return JSON.pretty_generate(:ktpid => params[:ktpid]) + # returns task details elsif params[:predictions] task = Task.find(params[:predictions]) pageSize = params[:pageSize].to_i - 1 -- cgit v1.2.3 From 62a7197574267404cc7034d78f552c4be0290a25 Mon Sep 17 00:00:00 2001 From: gebele Date: Fri, 2 Aug 2019 07:50:28 +0000 Subject: refresh batch dataset in task wait loop --- application.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 8872e5f..1c738f1 100644 --- a/application.rb +++ b/application.rb @@ -204,6 +204,7 @@ post '/predict/?' do input = Dataset.find_by(:source => "tmp/"+@filename) until input sleep 1 + input = Dataset.find_by(:source => "tmp/"+@filename) end prediction_dataset = model.predict input t.update_percent(70) -- cgit v1.2.3 From b349cb51c4e46e1ae1bb309e9c589f1cff89e5af Mon Sep 17 00:00:00 2001 From: gebele Date: Fri, 2 Aug 2019 10:57:06 +0000 Subject: optimize upload task with wait for dataset id --- application.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 1c738f1..965b7ec 100644 --- a/application.rb +++ b/application.rb @@ -180,14 +180,14 @@ post '/predict/?' do uploadDataset = Task.run do t = uploadTask t.update_percent(1) - $logger.debug "Processing '#{params[:fileselect][:filename]}'" - @input = Dataset.from_csv_file File.join("tmp", params[:fileselect][:filename]) + puts "Processing '#{params[:fileselect][:filename]}'" + input = Dataset.from_csv_file File.join("tmp", params[:fileselect][:filename]) + t[:dataset_id] = input.id t.update_percent(100) t.save end @upid = uploadTask.id - #TODO route for compound size @compounds_size = 0 #@input.compounds.size @models = params[:selection].keys @tasks = [] @@ -201,12 +201,12 @@ post '/predict/?' do prediction = {} model = Model::Validation.find model_id t.update_percent(10) - input = Dataset.find_by(:source => "tmp/"+@filename) - until input + until uploadTask.dataset_id sleep 1 - input = Dataset.find_by(:source => "tmp/"+@filename) + uploadTask = Task.find @upid end - prediction_dataset = model.predict input + @input = Dataset.find uploadTask.dataset_id + prediction_dataset = model.predict @input t.update_percent(70) t[:dataset_id] = prediction_dataset.id t.update_percent(75) -- cgit v1.2.3 From 498ad82d2cc8582d3139bf69a0fe333d6b425668 Mon Sep 17 00:00:00 2001 From: gebele Date: Thu, 8 Aug 2019 12:44:29 +0000 Subject: ensure dataset parser errors not hidden in a task; check upload by first header; remove_task_data, tasks, prediction dataset, training dataset; js code refinement --- application.rb | 70 +++++++++++++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 30 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 965b7ec..246efe1 100644 --- a/application.rb +++ b/application.rb @@ -67,13 +67,21 @@ not_found do end error do + # API errors if request.path.split("/")[1] == "api" || $paths.include?(request.path.split("/")[2]) @accept = request.env['HTTP_ACCEPT'] response['Content-Type'] = @accept @accept == "text/plain" ? request.env['sinatra.error'] : request.env['sinatra.error'].to_json + # batch dataset error + elsif request.env['sinatra.error.params']['batchfile'] && request.env['REQUEST_METHOD'] == "POST" + @error = request.env['sinatra.error'] + response['Content-Type'] = "text/html" + status 200 + return haml :error + # basic error else @error = request.env['sinatra.error'] - haml :error + return haml :error end end @@ -85,13 +93,17 @@ options "*" do end get '/predict/?' do + # handle user click on back button while batch prediction if params[:tpid] begin Process.kill(9,params[:tpid].to_i) if !params[:tpid].blank? rescue nil end + # remove data helper method + remove_task_data(params[:tpid]) end + # regular request on '/predict' page @models = OpenTox::Model::Validation.all @endpoints = @models.collect{|m| m.endpoint}.sort.uniq @models.count > 0 ? (haml :predict) : (haml :info) @@ -169,27 +181,31 @@ post '/predict/?' do # process batch prediction unless params[:fileselect].blank? if params[:fileselect][:filename] !~ /\.csv$/ - bad_request_error "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file." + raise "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file." end @filename = params[:fileselect][:filename] File.open('tmp/' + params[:fileselect][:filename], "w") do |f| f.write(params[:fileselect][:tempfile].read) end - uploadTask = Task.new - uploadTask.save - uploadDataset = Task.run do - t = uploadTask - t.update_percent(1) - puts "Processing '#{params[:fileselect][:filename]}'" - input = Dataset.from_csv_file File.join("tmp", params[:fileselect][:filename]) - t[:dataset_id] = input.id - t.update_percent(100) - t.save - end - @upid = uploadTask.id + # check CSV structure by parsing and header check + csv = CSV.read File.join("tmp", @filename) + header = csv.shift + accepted = ["SMILES","InChI"] + raise "CSV header does not include 'SMILES' or 'InChI'. Please read the HELP page." unless header.any?(/smiles|inchi/i) + @models = params[:selection].keys.join(",") + return haml :upload + end - @compounds_size = 0 #@input.compounds.size - @models = params[:selection].keys + unless params[:batchfile].blank? + dataset = Dataset.from_csv_file File.join("tmp", params[:batchfile]) + response['Content-Type'] = "application/json" + return {:dataset_id => dataset.id.to_s, :models => params[:models]}.to_json + end + + unless params[:models].blank? + dataset = Dataset.find params[:dataset_id] + @compounds_size = dataset.compounds.size + @models = params[:models].split(",") @tasks = [] @models.each{|m| t = Task.new; t.save; @tasks << t} @predictions = {} @@ -201,12 +217,7 @@ post '/predict/?' do prediction = {} model = Model::Validation.find model_id t.update_percent(10) - until uploadTask.dataset_id - sleep 1 - uploadTask = Task.find @upid - end - @input = Dataset.find uploadTask.dataset_id - prediction_dataset = model.predict @input + prediction_dataset = model.predict dataset t.update_percent(70) t[:dataset_id] = prediction_dataset.id t.update_percent(75) @@ -219,7 +230,11 @@ post '/predict/?' do t.save end end + maintask[:subTasks] = @tasks.collect{|t| t.id} + maintask.save @pid = maintask.pid + File.delete File.join(dataset.source) + response['Content-Type'] = "text/html" return haml :batch else # single compound prediction @@ -248,17 +263,11 @@ post '/predict/?' do end get '/prediction/task/?' do - # returns task progress in percentage + # returns task progress in percent if params[:turi] task = Task.find(params[:turi].to_s) response['Content-Type'] = "application/json" - if task.dataset_id - d = Dataset.find task.dataset_id - size = d.compounds.size - return JSON.pretty_generate(:percent => task.percent, :size => size) - else - return JSON.pretty_generate(:percent => task.percent) - end + return JSON.pretty_generate(:percent => task.percent) # kills task process id elsif params[:ktpid] begin @@ -266,6 +275,7 @@ get '/prediction/task/?' do rescue nil end + #remove_task_data(params[:ktpid]) deletes also the source file response['Content-Type'] = "application/json" return JSON.pretty_generate(:ktpid => params[:ktpid]) # returns task details -- cgit v1.2.3 From 74332693fed1d5ecb695dd6ae0c2fc611d14c4d7 Mon Sep 17 00:00:00 2001 From: gebele Date: Fri, 9 Aug 2019 12:28:19 +0000 Subject: do not delete batch tr_dataset if one of model tr_dataset;error message if dataset has no compounds --- application.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 246efe1..064c488 100644 --- a/application.rb +++ b/application.rb @@ -184,7 +184,7 @@ post '/predict/?' do raise "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file." end @filename = params[:fileselect][:filename] - File.open('tmp/' + params[:fileselect][:filename], "w") do |f| + File.open('tmp/' + @filename, "w") do |f| f.write(params[:fileselect][:tempfile].read) end # check CSV structure by parsing and header check @@ -198,6 +198,7 @@ post '/predict/?' do unless params[:batchfile].blank? dataset = Dataset.from_csv_file File.join("tmp", params[:batchfile]) + raise "No compounds in Dataset. Please read the HELP page." if dataset.compounds.size == 0 response['Content-Type'] = "application/json" return {:dataset_id => dataset.id.to_s, :models => params[:models]}.to_json end @@ -233,7 +234,6 @@ post '/predict/?' do maintask[:subTasks] = @tasks.collect{|t| t.id} maintask.save @pid = maintask.pid - File.delete File.join(dataset.source) response['Content-Type'] = "text/html" return haml :batch else -- cgit v1.2.3 From 270bddf5081671cd1905d6cce6eb3659159f573a Mon Sep 17 00:00:00 2001 From: gebele Date: Fri, 16 Aug 2019 09:49:53 +0000 Subject: change hostname method --- application.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'application.rb') diff --git a/application.rb b/application.rb index 064c488..d6cf0f8 100644 --- a/application.rb +++ b/application.rb @@ -40,6 +40,9 @@ configure :production, :development do end before do + # use this hostname method instead to('/') + # allowes to set https for xhr requests + $host_with_port = request.host =~ /localhost/ ? request.host_with_port : request.host $paths = [ "api", "compound", -- cgit v1.2.3 From 1e8179e995a82b11ec03e40e76ad155fe2cc2304 Mon Sep 17 00:00:00 2001 From: Christoph Helma Date: Thu, 29 Aug 2019 23:01:12 +0200 Subject: improved production settings --- application.rb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index d6cf0f8..a474782 100644 --- a/application.rb +++ b/application.rb @@ -18,7 +18,7 @@ PUBCHEM_CID_URI = PUBCHEM_URI.split("/")[0..-3].join("/")+"/compound/" "validation.rb" ].each{ |f| require_relative "./lib/#{f}" } -configure :production, :development do +configure :development do STDOUT.sync = true $logger = Logger.new(STDOUT) $logger.level = Logger::DEBUG @@ -39,6 +39,23 @@ configure :production, :development do ].each{ |f| also_reload "./lib/#{f}" } end +configure :production do + STDOUT.sync = true + $logger = Logger.new(STDOUT) + [ + "api.rb", + "compound.rb", + "dataset.rb", + "endpoint.rb", + "feature.rb", + "model.rb", + "report.rb", + "substance.rb", + "swagger.rb", + "validation.rb" + ].each{ |f| also_reload "./lib/#{f}" } +end + before do # use this hostname method instead to('/') # allowes to set https for xhr requests -- cgit v1.2.3 From 1144a04244df507b0ef64e6ee1cc49af70f06f3d Mon Sep 17 00:00:00 2001 From: Christoph Helma Date: Thu, 29 Aug 2019 23:15:48 +0200 Subject: another reload_also statement removed from production --- application.rb | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index a474782..fdf1f27 100644 --- a/application.rb +++ b/application.rb @@ -42,18 +42,6 @@ end configure :production do STDOUT.sync = true $logger = Logger.new(STDOUT) - [ - "api.rb", - "compound.rb", - "dataset.rb", - "endpoint.rb", - "feature.rb", - "model.rb", - "report.rb", - "substance.rb", - "swagger.rb", - "validation.rb" - ].each{ |f| also_reload "./lib/#{f}" } end before do -- cgit v1.2.3 From 1577e6db1eef50344b5416645a83539b02ed1b7e Mon Sep 17 00:00:00 2001 From: Christoph Helma Date: Thu, 29 Aug 2019 23:27:22 +0200 Subject: all also_reload calls removed --- application.rb | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index fdf1f27..ef059b2 100644 --- a/application.rb +++ b/application.rb @@ -18,30 +18,10 @@ PUBCHEM_CID_URI = PUBCHEM_URI.split("/")[0..-3].join("/")+"/compound/" "validation.rb" ].each{ |f| require_relative "./lib/#{f}" } -configure :development do +configure :production, :development do STDOUT.sync = true $logger = Logger.new(STDOUT) $logger.level = Logger::DEBUG - enable :reloader - also_reload './helper.rb' - also_reload './qmrf_report.rb' - [ - "api.rb", - "compound.rb", - "dataset.rb", - "endpoint.rb", - "feature.rb", - "model.rb", - "report.rb", - "substance.rb", - "swagger.rb", - "validation.rb" - ].each{ |f| also_reload "./lib/#{f}" } -end - -configure :production do - STDOUT.sync = true - $logger = Logger.new(STDOUT) end before do -- cgit v1.2.3 From c9f8cd864a2b72e7330b8095691fa5c16fea83aa Mon Sep 17 00:00:00 2001 From: Christoph Helma Date: Fri, 30 Aug 2019 11:14:50 +0200 Subject: Dockerfile and start script added --- application.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index ef059b2..fb03a72 100644 --- a/application.rb +++ b/application.rb @@ -21,7 +21,7 @@ PUBCHEM_CID_URI = PUBCHEM_URI.split("/")[0..-3].join("/")+"/compound/" configure :production, :development do STDOUT.sync = true $logger = Logger.new(STDOUT) - $logger.level = Logger::DEBUG + #$logger.level = Logger::DEBUG end before do @@ -42,7 +42,7 @@ before do if request.path =~ /predict/ @accept = request.env['HTTP_ACCEPT'].split(",").first response['Content-Type'] = @accept - halt 400, "Mime type #{@accept} is not supported." unless @accept == "text/html" || "*/*" + halt 400, "Mime type #{@accept} is not supported." unless @accept == "text/html" or @accept == "*/*" @version = File.read("VERSION").chomp else @accept = request.env['HTTP_ACCEPT'].split(",").first -- cgit v1.2.3 From a84d9eabf1b921086a688f81df28b0f21ba4df19 Mon Sep 17 00:00:00 2001 From: Christoph Helma Date: Tue, 3 Sep 2019 13:06:00 +0200 Subject: docker installation as gem, links with port numbers, version 1.4.0 --- application.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'application.rb') diff --git a/application.rb b/application.rb index fb03a72..e23c042 100644 --- a/application.rb +++ b/application.rb @@ -18,16 +18,22 @@ PUBCHEM_CID_URI = PUBCHEM_URI.split("/")[0..-3].join("/")+"/compound/" "validation.rb" ].each{ |f| require_relative "./lib/#{f}" } -configure :production, :development do +configure :production do STDOUT.sync = true $logger = Logger.new(STDOUT) - #$logger.level = Logger::DEBUG +end + +configure :development do + STDOUT.sync = true + $logger = Logger.new(STDOUT) + $logger.level = Logger::DEBUG end before do # use this hostname method instead to('/') # allowes to set https for xhr requests - $host_with_port = request.host =~ /localhost/ ? request.host_with_port : request.host + #$host_with_port = request.host =~ /localhost/ ? request.host_with_port : request.host + $host_with_port = request.host_with_port $paths = [ "api", "compound", -- cgit v1.2.3