diff options
author | gebele <gebele@in-silico.ch> | 2019-08-08 12:44:29 +0000 |
---|---|---|
committer | gebele <gebele@in-silico.ch> | 2019-08-08 12:44:29 +0000 |
commit | 498ad82d2cc8582d3139bf69a0fe333d6b425668 (patch) | |
tree | a660e70ddb709e4e20ff6a70f952f7d07e7c56fe | |
parent | 185a6df5e09dc89a50f23858e9cb221aacca9327 (diff) |
ensure dataset parser errors not hidden in a task; check upload by first header; remove_task_data, tasks, prediction dataset, training dataset; js code refinement
-rw-r--r-- | application.rb | 70 | ||||
-rw-r--r-- | helper.rb | 21 | ||||
-rw-r--r-- | public/javascripts/lazar-gui.js | 110 | ||||
-rw-r--r-- | task.rb | 7 | ||||
-rw-r--r-- | views/batch.haml | 44 | ||||
-rw-r--r-- | views/upload.haml | 28 |
6 files changed, 168 insertions, 112 deletions
diff --git a/application.rb b/application.rb index 965b7ec..246efe1 100644 --- a/application.rb +++ b/application.rb @@ -67,13 +67,21 @@ not_found do end error do + # API errors if request.path.split("/")[1] == "api" || $paths.include?(request.path.split("/")[2]) @accept = request.env['HTTP_ACCEPT'] response['Content-Type'] = @accept @accept == "text/plain" ? request.env['sinatra.error'] : request.env['sinatra.error'].to_json + # batch dataset error + elsif request.env['sinatra.error.params']['batchfile'] && request.env['REQUEST_METHOD'] == "POST" + @error = request.env['sinatra.error'] + response['Content-Type'] = "text/html" + status 200 + return haml :error + # basic error else @error = request.env['sinatra.error'] - haml :error + return haml :error end end @@ -85,13 +93,17 @@ options "*" do end get '/predict/?' do + # handle user click on back button while batch prediction if params[:tpid] begin Process.kill(9,params[:tpid].to_i) if !params[:tpid].blank? rescue nil end + # remove data helper method + remove_task_data(params[:tpid]) end + # regular request on '/predict' page @models = OpenTox::Model::Validation.all @endpoints = @models.collect{|m| m.endpoint}.sort.uniq @models.count > 0 ? (haml :predict) : (haml :info) @@ -169,27 +181,31 @@ post '/predict/?' do # process batch prediction unless params[:fileselect].blank? if params[:fileselect][:filename] !~ /\.csv$/ - bad_request_error "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file." + raise "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file." end @filename = params[:fileselect][:filename] File.open('tmp/' + params[:fileselect][:filename], "w") do |f| f.write(params[:fileselect][:tempfile].read) end - uploadTask = Task.new - uploadTask.save - uploadDataset = Task.run do - t = uploadTask - t.update_percent(1) - puts "Processing '#{params[:fileselect][:filename]}'" - input = Dataset.from_csv_file File.join("tmp", params[:fileselect][:filename]) - t[:dataset_id] = input.id - t.update_percent(100) - t.save - end - @upid = uploadTask.id + # check CSV structure by parsing and header check + csv = CSV.read File.join("tmp", @filename) + header = csv.shift + accepted = ["SMILES","InChI"] + raise "CSV header does not include 'SMILES' or 'InChI'. Please read the <a href='https://dg.in-silico.ch/predict/help' rel='external'> HELP </a> page." unless header.any?(/smiles|inchi/i) + @models = params[:selection].keys.join(",") + return haml :upload + end - @compounds_size = 0 #@input.compounds.size - @models = params[:selection].keys + unless params[:batchfile].blank? + dataset = Dataset.from_csv_file File.join("tmp", params[:batchfile]) + response['Content-Type'] = "application/json" + return {:dataset_id => dataset.id.to_s, :models => params[:models]}.to_json + end + + unless params[:models].blank? + dataset = Dataset.find params[:dataset_id] + @compounds_size = dataset.compounds.size + @models = params[:models].split(",") @tasks = [] @models.each{|m| t = Task.new; t.save; @tasks << t} @predictions = {} @@ -201,12 +217,7 @@ post '/predict/?' do prediction = {} model = Model::Validation.find model_id t.update_percent(10) - until uploadTask.dataset_id - sleep 1 - uploadTask = Task.find @upid - end - @input = Dataset.find uploadTask.dataset_id - prediction_dataset = model.predict @input + prediction_dataset = model.predict dataset t.update_percent(70) t[:dataset_id] = prediction_dataset.id t.update_percent(75) @@ -219,7 +230,11 @@ post '/predict/?' do t.save end end + maintask[:subTasks] = @tasks.collect{|t| t.id} + maintask.save @pid = maintask.pid + File.delete File.join(dataset.source) + response['Content-Type'] = "text/html" return haml :batch else # single compound prediction @@ -248,17 +263,11 @@ post '/predict/?' do end get '/prediction/task/?' do - # returns task progress in percentage + # returns task progress in percent if params[:turi] task = Task.find(params[:turi].to_s) response['Content-Type'] = "application/json" - if task.dataset_id - d = Dataset.find task.dataset_id - size = d.compounds.size - return JSON.pretty_generate(:percent => task.percent, :size => size) - else - return JSON.pretty_generate(:percent => task.percent) - end + return JSON.pretty_generate(:percent => task.percent) # kills task process id elsif params[:ktpid] begin @@ -266,6 +275,7 @@ get '/prediction/task/?' do rescue nil end + #remove_task_data(params[:ktpid]) deletes also the source file response['Content-Type'] = "application/json" return JSON.pretty_generate(:ktpid => params[:ktpid]) # returns task details @@ -23,4 +23,25 @@ helpers do self.match(/^[a-f\d]{24}$/i) ? true : false end + def remove_task_data(pid) + task = Task.find_by(:pid => pid) + if task and !task.subTasks.blank? + task.subTasks.each_with_index do |task_id,idx| + t = Task.find task_id + predictionDataset = Dataset.find t.dataset_id if t.dataset_id + if predictionDataset && idx == 0 + trainingDataset = Dataset.find predictionDataset.source + source = trainingDataset.source + trainingDataset.delete + File.delete File.join(source) if File.exists? File.join(source) + predictionDataset.delete + elsif predictionDataset + predictionDataset.delete + end + t.delete + end + end + task.delete if task + end + end diff --git a/public/javascripts/lazar-gui.js b/public/javascripts/lazar-gui.js index 724262e..1899886 100644 --- a/public/javascripts/lazar-gui.js +++ b/public/javascripts/lazar-gui.js @@ -33,6 +33,20 @@ var HttpClient = function() { anHttpRequest.open( "GET", aUrl, true ); anHttpRequest.send( null ); } + this.post = function(aUrl, params, aCallback) { + var anHttpRequest = new XMLHttpRequest(); + anHttpRequest.onreadystatechange = function() { + //alert(anHttpRequest.status); + if (anHttpRequest.readyState == 4 && anHttpRequest.status == 200) + aCallback(anHttpRequest); + } + if (anHttpRequest.readyState == 4 && anHttpRequest.status == 400){ + aCallback(anHttpRequest); + } + anHttpRequest.open( "POST", aUrl, true ); + anHttpRequest.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); + anHttpRequest.send( params ); + } }; // functions used in predict.haml @@ -240,6 +254,29 @@ renderTask = function(task_uri,id) { }); }; +taskProgress = function(idx,timer,task_uri){ + // wait until previous task is completed + if (idx > 0){ + markers[idx] = setInterval(function(){ + var button = document.getElementById("detailsbutton_"+(idx-1)); + if(!button.classList.contains('disabled')){ + renderTask(task_uri,idx); + $("#est_"+idx).hide(); + $("#circle_"+idx).show(); + } + }, timer ); + }else{ + if ( $("#est_"+idx).is(":visible") ){ + $("#est_"+idx).hide(); + $("#circle_"+idx).show(); + } + renderTask(task_uri,idx); + markers[idx] = setInterval(function(){ + renderTask(task_uri,idx); + }, timer ); + }; +}; + simpleTemplating = function(data) { var html = '<ul class=pagination>'; $.each(data, function(index, item){ @@ -249,48 +286,39 @@ simpleTemplating = function(data) { return html; }; -pagePredictions = function(task_uri,model_id,id, compounds_uri){ +pagePredictions = function(task_uri,model_id,id, compoundsSize){ button = document.getElementById("detailsbutton_"+id); span = button.childNodes[1]; - // get compounds size first - var compoundsSize = 0; - var aClient = new HttpClient(); - aClient.get(compounds_uri, function(res) { - var response = JSON.parse(res); - compoundsSize = response['size'] - // handle pager in callback function - // ensures compoundsSize is > 0 - if (span.className == "fa fa-caret-right"){ - span.className = "fa fa-caret-down"; - $('#data-container_'+id).removeClass("d-none"); - $('#data-container_'+id).show(); - $('#pager_'+id).show(); - $('#pager_'+id).pagination({ - dataSource: task_uri, - locator: 'prediction', - totalNumber: compoundsSize, - pageSize: 1, - showPageNumbers: true, - showGoInput: true, - formatGoInput: 'go to <%= input %>', - formatAjaxError: function(jqXHR, textStatus, errorThrown) { - $('#data-container_'+id).html(errorThrown); - }, - /*ajax: { - beforeSend: function() { - $('#data-container_'+id).html('Loading content ...'); - } - },*/ - callback: function(data, pagination) { - var html = simpleTemplating(data); - $('#data-container_'+id).html(html); - //$('#data-container_'+id).css("min-height", $(window).height() + "px" ); + if (span.className == "fa fa-caret-right"){ + span.className = "fa fa-caret-down"; + $('#data-container_'+id).removeClass("d-none"); + $('#data-container_'+id).show(); + $('#pager_'+id).show(); + $('#pager_'+id).pagination({ + dataSource: task_uri, + locator: 'prediction', + totalNumber: compoundsSize, + pageSize: 1, + showPageNumbers: true, + showGoInput: true, + formatGoInput: 'go to <%= input %>', + formatAjaxError: function(jqXHR, textStatus, errorThrown) { + $('#data-container_'+id).html(errorThrown); + }, + ajax: { + beforeSend: function() { + $('#data-container_'+id).html('Loading content ...'); } - }); - } else if (span.className = "fa fa-caret-down"){ - span.className = "fa fa-caret-right"; - $('#data-container_'+id).hide(); - $('#pager_'+id).hide(); - }; - }); + }, + callback: function(data, pagination) { + var html = simpleTemplating(data); + $('#data-container_'+id).html(html); + //$('#data-container_'+id).css("min-height", $(window).height() + "px" ); + } + }); + } else if (span.className = "fa fa-caret-down"){ + span.className = "fa fa-caret-right"; + $('#data-container_'+id).hide(); + $('#pager_'+id).hide(); + }; }; @@ -14,8 +14,9 @@ module OpenTox field :csv, type: String field :dataset_id, type: BSON::ObjectId field :model_id, type: BSON::ObjectId + field :subTasks, type: Array, default:[] - attr_accessor :pid, :percent, :predictions, :csv, :dataset_id, :model_id + attr_accessor :pid, :percent, :predictions, :csv, :dataset_id, :model_id, :subTasks def pid self[:pid] @@ -41,6 +42,10 @@ module OpenTox self[:model_id] end + def subTasks + self[:subTasks] + end + def update_percent(percent) self[:percent] = percent save diff --git a/views/batch.haml b/views/batch.haml index e0fc573..4e1df05 100644 --- a/views/batch.haml +++ b/views/batch.haml @@ -11,22 +11,13 @@ %a.btn.btn-outline-info{:href => "//#{ENV['VIRTUAL_HOST']}/predict?tpid=#{@pid}"} %span.fa.fa-caret-left{:aria=>{:hidden=>"true"}} New Prediction -%div.card.bg-light{:id=>"uploadDataset"} - %div.card-body - %h3.card-title="Processing file #{@filename} to dataset." - %img.h2{:src=>"/images/wait30trans.gif", :id=>"circle_upload", :class=>"circle", :alt=>"processing"} - :javascript - uploadInterval = setInterval(function(){ - uploadDataset('//#{ENV['VIRTUAL_HOST']}/prediction/task/?turi=#{@upid}'); - }, 1000 ); - %div.card.bg-light %div.card-body - %h3.card-title="Batch prediction results for: #{@filename}" + %h3.card-title="Batch prediction results for: #{@filename}" // prepare variable values for javascript // increase timer interval for large datasets - - ctimer = 10000#((@compounds_size/1000) == 0 ? 1000 : ((@compounds_size/1000)*1000)) + - timer = 10000#((@compounds_size/1000) == 0 ? 1000 : ((@compounds_size/1000)*1000)) // process batch predictions - @models.each_with_index do |model,idx| - m = Model::Validation.find model @@ -38,7 +29,7 @@ %h5.card-title="#{m.endpoint} (#{m.species})" #pager{:id=>idx} %div.col-6 - %a.btn.btn-outline-info.btn-sm.disabled{:id => "detailsbutton_#{idx}", :data=>{:toggle=>"collapse"}, :href=>"javascript:void(0)", :onclick=>"pagePredictions('//#{ENV['VIRTUAL_HOST']}/prediction/task/?predictions=#{task}','#{model}','#{idx}','//#{ENV['VIRTUAL_HOST']}/prediction/task/?turi=#{task}')"} + %a.btn.btn-outline-info.btn-sm.disabled{:id => "detailsbutton_#{idx}", :data=>{:toggle=>"collapse"}, :href=>"javascript:void(0)", :onclick=>"pagePredictions('//#{ENV['VIRTUAL_HOST']}/prediction/task/?predictions=#{task}','#{model}','#{idx}','#{@compounds_size}')"} %span.fa.fa-caret-right Details %a.btn.btn-outline-info.btn-sm.disabled{:id => "downbutton_#{idx}", :href=>"//#{ENV['VIRTUAL_HOST']}/predict/batch/download?tid=#{task}", :title=>"download"} @@ -49,37 +40,10 @@ %img.h2{:src=>"/images/wait30trans.gif", :id=>"circle_#{idx}", :class=>"circle", :alt=>"wait", :style=>"display:none;"} :javascript $(document).ready(function() { - taskProgress('#{idx}','#{ctimer}','//#{ENV['VIRTUAL_HOST']}/prediction/task/?turi=#{task}'); + taskProgress('#{idx}','#{timer}','//#{ENV['VIRTUAL_HOST']}/prediction/task/?turi=#{task}'); }); #data-container.card.d-none.table-responsive{:id=>idx} :javascript - taskProgress = function(idx,timer,task_uri){ - // wait until previous task is completed - if (idx > 0){ - markers[idx] = setInterval(function(){ - var button = document.getElementById("detailsbutton_"+(idx-1)); - if(!button.classList.contains('disabled')){ - renderTask(task_uri,idx); - $("#est_"+idx).hide(); - $("#circle_"+idx).show(); - } - }, timer ); - }else{ - markers[idx] = setInterval(function(){ - // check that dataset parsing is completed - if (document.getElementById("uploadDataset")){ - $("#est_"+idx).show(); - $("#circle_"+idx).hide(); - } else { - renderTask(task_uri,idx); - if ( $("#est_"+idx).is(":visible") ){ - $("#est_"+idx).hide(); - $("#circle_"+idx).show(); - } - } - }, timer ); - }; - }; %div.modal.fade{:id=>"details", :tabindex=>"-1", :role=>"dialog"} %div.modal-dialog.modal-lg{:role=>"document"} %div.modal-content diff --git a/views/upload.haml b/views/upload.haml new file mode 100644 index 0000000..159f7a5 --- /dev/null +++ b/views/upload.haml @@ -0,0 +1,28 @@ +%div.card + %a.btn.btn-outline-info{:href => "//#{ENV['VIRTUAL_HOST']}/predict"} + %span.fa.fa-caret-left{:aria=>{:hidden=>"true"}} + New Prediction +%div.card.bg-light{:id=>"uploadDataset"} + %div.card-body + %h3.card-title="Processing file #{@filename} to dataset." + %img.h2{:src=>"/images/wait30trans.gif", :id=>"circle_upload", :class=>"circle", :alt=>"processing"} + :javascript + $(document).ready(function() { + var aClient = new HttpClient(); + aClient.post('//#{ENV['VIRTUAL_HOST']}/predict', 'models=#{@models}&batchfile=#{@filename}', function(res1) { + var contentType = res1.getResponseHeader('content-type'); + if (contentType == "application/json"){ + var response = JSON.parse(res1.responseText); + } else { + var response = res1.responseText; + } + if (res1.status == 200 && response['models'] && response['dataset_id']){ + aClient.post('//#{ENV['VIRTUAL_HOST']}/predict', 'models='+response['models']+'&dataset_id='+response['dataset_id'], function(res2) { + $("body").html(res2.responseText); + }); + } + if (res1.status == 200 && contentType == "text/html"){ + $("body").html(response); + } + }); + }); |