summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorgebele <gebele@in-silico.ch>2019-08-08 12:44:29 +0000
committergebele <gebele@in-silico.ch>2019-08-08 12:44:29 +0000
commit498ad82d2cc8582d3139bf69a0fe333d6b425668 (patch)
treea660e70ddb709e4e20ff6a70f952f7d07e7c56fe
parent185a6df5e09dc89a50f23858e9cb221aacca9327 (diff)
ensure dataset parser errors not hidden in a task; check upload by first header; remove_task_data, tasks, prediction dataset, training dataset; js code refinement
-rw-r--r--application.rb70
-rw-r--r--helper.rb21
-rw-r--r--public/javascripts/lazar-gui.js110
-rw-r--r--task.rb7
-rw-r--r--views/batch.haml44
-rw-r--r--views/upload.haml28
6 files changed, 168 insertions, 112 deletions
diff --git a/application.rb b/application.rb
index 965b7ec..246efe1 100644
--- a/application.rb
+++ b/application.rb
@@ -67,13 +67,21 @@ not_found do
end
error do
+ # API errors
if request.path.split("/")[1] == "api" || $paths.include?(request.path.split("/")[2])
@accept = request.env['HTTP_ACCEPT']
response['Content-Type'] = @accept
@accept == "text/plain" ? request.env['sinatra.error'] : request.env['sinatra.error'].to_json
+ # batch dataset error
+ elsif request.env['sinatra.error.params']['batchfile'] && request.env['REQUEST_METHOD'] == "POST"
+ @error = request.env['sinatra.error']
+ response['Content-Type'] = "text/html"
+ status 200
+ return haml :error
+ # basic error
else
@error = request.env['sinatra.error']
- haml :error
+ return haml :error
end
end
@@ -85,13 +93,17 @@ options "*" do
end
get '/predict/?' do
+ # handle user click on back button while batch prediction
if params[:tpid]
begin
Process.kill(9,params[:tpid].to_i) if !params[:tpid].blank?
rescue
nil
end
+ # remove data helper method
+ remove_task_data(params[:tpid])
end
+ # regular request on '/predict' page
@models = OpenTox::Model::Validation.all
@endpoints = @models.collect{|m| m.endpoint}.sort.uniq
@models.count > 0 ? (haml :predict) : (haml :info)
@@ -169,27 +181,31 @@ post '/predict/?' do
# process batch prediction
unless params[:fileselect].blank?
if params[:fileselect][:filename] !~ /\.csv$/
- bad_request_error "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file."
+ raise "Wrong file extension for '#{params[:fileselect][:filename]}'. Please upload a CSV file."
end
@filename = params[:fileselect][:filename]
File.open('tmp/' + params[:fileselect][:filename], "w") do |f|
f.write(params[:fileselect][:tempfile].read)
end
- uploadTask = Task.new
- uploadTask.save
- uploadDataset = Task.run do
- t = uploadTask
- t.update_percent(1)
- puts "Processing '#{params[:fileselect][:filename]}'"
- input = Dataset.from_csv_file File.join("tmp", params[:fileselect][:filename])
- t[:dataset_id] = input.id
- t.update_percent(100)
- t.save
- end
- @upid = uploadTask.id
+ # check CSV structure by parsing and header check
+ csv = CSV.read File.join("tmp", @filename)
+ header = csv.shift
+ accepted = ["SMILES","InChI"]
+ raise "CSV header does not include 'SMILES' or 'InChI'. Please read the <a href='https://dg.in-silico.ch/predict/help' rel='external'> HELP </a> page." unless header.any?(/smiles|inchi/i)
+ @models = params[:selection].keys.join(",")
+ return haml :upload
+ end
- @compounds_size = 0 #@input.compounds.size
- @models = params[:selection].keys
+ unless params[:batchfile].blank?
+ dataset = Dataset.from_csv_file File.join("tmp", params[:batchfile])
+ response['Content-Type'] = "application/json"
+ return {:dataset_id => dataset.id.to_s, :models => params[:models]}.to_json
+ end
+
+ unless params[:models].blank?
+ dataset = Dataset.find params[:dataset_id]
+ @compounds_size = dataset.compounds.size
+ @models = params[:models].split(",")
@tasks = []
@models.each{|m| t = Task.new; t.save; @tasks << t}
@predictions = {}
@@ -201,12 +217,7 @@ post '/predict/?' do
prediction = {}
model = Model::Validation.find model_id
t.update_percent(10)
- until uploadTask.dataset_id
- sleep 1
- uploadTask = Task.find @upid
- end
- @input = Dataset.find uploadTask.dataset_id
- prediction_dataset = model.predict @input
+ prediction_dataset = model.predict dataset
t.update_percent(70)
t[:dataset_id] = prediction_dataset.id
t.update_percent(75)
@@ -219,7 +230,11 @@ post '/predict/?' do
t.save
end
end
+ maintask[:subTasks] = @tasks.collect{|t| t.id}
+ maintask.save
@pid = maintask.pid
+ File.delete File.join(dataset.source)
+ response['Content-Type'] = "text/html"
return haml :batch
else
# single compound prediction
@@ -248,17 +263,11 @@ post '/predict/?' do
end
get '/prediction/task/?' do
- # returns task progress in percentage
+ # returns task progress in percent
if params[:turi]
task = Task.find(params[:turi].to_s)
response['Content-Type'] = "application/json"
- if task.dataset_id
- d = Dataset.find task.dataset_id
- size = d.compounds.size
- return JSON.pretty_generate(:percent => task.percent, :size => size)
- else
- return JSON.pretty_generate(:percent => task.percent)
- end
+ return JSON.pretty_generate(:percent => task.percent)
# kills task process id
elsif params[:ktpid]
begin
@@ -266,6 +275,7 @@ get '/prediction/task/?' do
rescue
nil
end
+ #remove_task_data(params[:ktpid]) deletes also the source file
response['Content-Type'] = "application/json"
return JSON.pretty_generate(:ktpid => params[:ktpid])
# returns task details
diff --git a/helper.rb b/helper.rb
index cb77ffd..82482d4 100644
--- a/helper.rb
+++ b/helper.rb
@@ -23,4 +23,25 @@ helpers do
self.match(/^[a-f\d]{24}$/i) ? true : false
end
+ def remove_task_data(pid)
+ task = Task.find_by(:pid => pid)
+ if task and !task.subTasks.blank?
+ task.subTasks.each_with_index do |task_id,idx|
+ t = Task.find task_id
+ predictionDataset = Dataset.find t.dataset_id if t.dataset_id
+ if predictionDataset && idx == 0
+ trainingDataset = Dataset.find predictionDataset.source
+ source = trainingDataset.source
+ trainingDataset.delete
+ File.delete File.join(source) if File.exists? File.join(source)
+ predictionDataset.delete
+ elsif predictionDataset
+ predictionDataset.delete
+ end
+ t.delete
+ end
+ end
+ task.delete if task
+ end
+
end
diff --git a/public/javascripts/lazar-gui.js b/public/javascripts/lazar-gui.js
index 724262e..1899886 100644
--- a/public/javascripts/lazar-gui.js
+++ b/public/javascripts/lazar-gui.js
@@ -33,6 +33,20 @@ var HttpClient = function() {
anHttpRequest.open( "GET", aUrl, true );
anHttpRequest.send( null );
}
+ this.post = function(aUrl, params, aCallback) {
+ var anHttpRequest = new XMLHttpRequest();
+ anHttpRequest.onreadystatechange = function() {
+ //alert(anHttpRequest.status);
+ if (anHttpRequest.readyState == 4 && anHttpRequest.status == 200)
+ aCallback(anHttpRequest);
+ }
+ if (anHttpRequest.readyState == 4 && anHttpRequest.status == 400){
+ aCallback(anHttpRequest);
+ }
+ anHttpRequest.open( "POST", aUrl, true );
+ anHttpRequest.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
+ anHttpRequest.send( params );
+ }
};
// functions used in predict.haml
@@ -240,6 +254,29 @@ renderTask = function(task_uri,id) {
});
};
+taskProgress = function(idx,timer,task_uri){
+ // wait until previous task is completed
+ if (idx > 0){
+ markers[idx] = setInterval(function(){
+ var button = document.getElementById("detailsbutton_"+(idx-1));
+ if(!button.classList.contains('disabled')){
+ renderTask(task_uri,idx);
+ $("#est_"+idx).hide();
+ $("#circle_"+idx).show();
+ }
+ }, timer );
+ }else{
+ if ( $("#est_"+idx).is(":visible") ){
+ $("#est_"+idx).hide();
+ $("#circle_"+idx).show();
+ }
+ renderTask(task_uri,idx);
+ markers[idx] = setInterval(function(){
+ renderTask(task_uri,idx);
+ }, timer );
+ };
+};
+
simpleTemplating = function(data) {
var html = '<ul class=pagination>';
$.each(data, function(index, item){
@@ -249,48 +286,39 @@ simpleTemplating = function(data) {
return html;
};
-pagePredictions = function(task_uri,model_id,id, compounds_uri){
+pagePredictions = function(task_uri,model_id,id, compoundsSize){
button = document.getElementById("detailsbutton_"+id);
span = button.childNodes[1];
- // get compounds size first
- var compoundsSize = 0;
- var aClient = new HttpClient();
- aClient.get(compounds_uri, function(res) {
- var response = JSON.parse(res);
- compoundsSize = response['size']
- // handle pager in callback function
- // ensures compoundsSize is > 0
- if (span.className == "fa fa-caret-right"){
- span.className = "fa fa-caret-down";
- $('#data-container_'+id).removeClass("d-none");
- $('#data-container_'+id).show();
- $('#pager_'+id).show();
- $('#pager_'+id).pagination({
- dataSource: task_uri,
- locator: 'prediction',
- totalNumber: compoundsSize,
- pageSize: 1,
- showPageNumbers: true,
- showGoInput: true,
- formatGoInput: 'go to <%= input %>',
- formatAjaxError: function(jqXHR, textStatus, errorThrown) {
- $('#data-container_'+id).html(errorThrown);
- },
- /*ajax: {
- beforeSend: function() {
- $('#data-container_'+id).html('Loading content ...');
- }
- },*/
- callback: function(data, pagination) {
- var html = simpleTemplating(data);
- $('#data-container_'+id).html(html);
- //$('#data-container_'+id).css("min-height", $(window).height() + "px" );
+ if (span.className == "fa fa-caret-right"){
+ span.className = "fa fa-caret-down";
+ $('#data-container_'+id).removeClass("d-none");
+ $('#data-container_'+id).show();
+ $('#pager_'+id).show();
+ $('#pager_'+id).pagination({
+ dataSource: task_uri,
+ locator: 'prediction',
+ totalNumber: compoundsSize,
+ pageSize: 1,
+ showPageNumbers: true,
+ showGoInput: true,
+ formatGoInput: 'go to <%= input %>',
+ formatAjaxError: function(jqXHR, textStatus, errorThrown) {
+ $('#data-container_'+id).html(errorThrown);
+ },
+ ajax: {
+ beforeSend: function() {
+ $('#data-container_'+id).html('Loading content ...');
}
- });
- } else if (span.className = "fa fa-caret-down"){
- span.className = "fa fa-caret-right";
- $('#data-container_'+id).hide();
- $('#pager_'+id).hide();
- };
- });
+ },
+ callback: function(data, pagination) {
+ var html = simpleTemplating(data);
+ $('#data-container_'+id).html(html);
+ //$('#data-container_'+id).css("min-height", $(window).height() + "px" );
+ }
+ });
+ } else if (span.className = "fa fa-caret-down"){
+ span.className = "fa fa-caret-right";
+ $('#data-container_'+id).hide();
+ $('#pager_'+id).hide();
+ };
};
diff --git a/task.rb b/task.rb
index 708613e..20dcc68 100644
--- a/task.rb
+++ b/task.rb
@@ -14,8 +14,9 @@ module OpenTox
field :csv, type: String
field :dataset_id, type: BSON::ObjectId
field :model_id, type: BSON::ObjectId
+ field :subTasks, type: Array, default:[]
- attr_accessor :pid, :percent, :predictions, :csv, :dataset_id, :model_id
+ attr_accessor :pid, :percent, :predictions, :csv, :dataset_id, :model_id, :subTasks
def pid
self[:pid]
@@ -41,6 +42,10 @@ module OpenTox
self[:model_id]
end
+ def subTasks
+ self[:subTasks]
+ end
+
def update_percent(percent)
self[:percent] = percent
save
diff --git a/views/batch.haml b/views/batch.haml
index e0fc573..4e1df05 100644
--- a/views/batch.haml
+++ b/views/batch.haml
@@ -11,22 +11,13 @@
%a.btn.btn-outline-info{:href => "//#{ENV['VIRTUAL_HOST']}/predict?tpid=#{@pid}"}
%span.fa.fa-caret-left{:aria=>{:hidden=>"true"}}
New Prediction
-%div.card.bg-light{:id=>"uploadDataset"}
- %div.card-body
- %h3.card-title="Processing file #{@filename} to dataset."
- %img.h2{:src=>"/images/wait30trans.gif", :id=>"circle_upload", :class=>"circle", :alt=>"processing"}
- :javascript
- uploadInterval = setInterval(function(){
- uploadDataset('//#{ENV['VIRTUAL_HOST']}/prediction/task/?turi=#{@upid}');
- }, 1000 );
-
%div.card.bg-light
%div.card-body
- %h3.card-title="Batch prediction results for: #{@filename}"
+ %h3.card-title="Batch prediction results for: #{@filename}"
// prepare variable values for javascript
// increase timer interval for large datasets
- - ctimer = 10000#((@compounds_size/1000) == 0 ? 1000 : ((@compounds_size/1000)*1000))
+ - timer = 10000#((@compounds_size/1000) == 0 ? 1000 : ((@compounds_size/1000)*1000))
// process batch predictions
- @models.each_with_index do |model,idx|
- m = Model::Validation.find model
@@ -38,7 +29,7 @@
%h5.card-title="#{m.endpoint} (#{m.species})"
#pager{:id=>idx}
%div.col-6
- %a.btn.btn-outline-info.btn-sm.disabled{:id => "detailsbutton_#{idx}", :data=>{:toggle=>"collapse"}, :href=>"javascript:void(0)", :onclick=>"pagePredictions('//#{ENV['VIRTUAL_HOST']}/prediction/task/?predictions=#{task}','#{model}','#{idx}','//#{ENV['VIRTUAL_HOST']}/prediction/task/?turi=#{task}')"}
+ %a.btn.btn-outline-info.btn-sm.disabled{:id => "detailsbutton_#{idx}", :data=>{:toggle=>"collapse"}, :href=>"javascript:void(0)", :onclick=>"pagePredictions('//#{ENV['VIRTUAL_HOST']}/prediction/task/?predictions=#{task}','#{model}','#{idx}','#{@compounds_size}')"}
%span.fa.fa-caret-right
Details
%a.btn.btn-outline-info.btn-sm.disabled{:id => "downbutton_#{idx}", :href=>"//#{ENV['VIRTUAL_HOST']}/predict/batch/download?tid=#{task}", :title=>"download"}
@@ -49,37 +40,10 @@
%img.h2{:src=>"/images/wait30trans.gif", :id=>"circle_#{idx}", :class=>"circle", :alt=>"wait", :style=>"display:none;"}
:javascript
$(document).ready(function() {
- taskProgress('#{idx}','#{ctimer}','//#{ENV['VIRTUAL_HOST']}/prediction/task/?turi=#{task}');
+ taskProgress('#{idx}','#{timer}','//#{ENV['VIRTUAL_HOST']}/prediction/task/?turi=#{task}');
});
#data-container.card.d-none.table-responsive{:id=>idx}
:javascript
- taskProgress = function(idx,timer,task_uri){
- // wait until previous task is completed
- if (idx > 0){
- markers[idx] = setInterval(function(){
- var button = document.getElementById("detailsbutton_"+(idx-1));
- if(!button.classList.contains('disabled')){
- renderTask(task_uri,idx);
- $("#est_"+idx).hide();
- $("#circle_"+idx).show();
- }
- }, timer );
- }else{
- markers[idx] = setInterval(function(){
- // check that dataset parsing is completed
- if (document.getElementById("uploadDataset")){
- $("#est_"+idx).show();
- $("#circle_"+idx).hide();
- } else {
- renderTask(task_uri,idx);
- if ( $("#est_"+idx).is(":visible") ){
- $("#est_"+idx).hide();
- $("#circle_"+idx).show();
- }
- }
- }, timer );
- };
- };
%div.modal.fade{:id=>"details", :tabindex=>"-1", :role=>"dialog"}
%div.modal-dialog.modal-lg{:role=>"document"}
%div.modal-content
diff --git a/views/upload.haml b/views/upload.haml
new file mode 100644
index 0000000..159f7a5
--- /dev/null
+++ b/views/upload.haml
@@ -0,0 +1,28 @@
+%div.card
+ %a.btn.btn-outline-info{:href => "//#{ENV['VIRTUAL_HOST']}/predict"}
+ %span.fa.fa-caret-left{:aria=>{:hidden=>"true"}}
+ New Prediction
+%div.card.bg-light{:id=>"uploadDataset"}
+ %div.card-body
+ %h3.card-title="Processing file #{@filename} to dataset."
+ %img.h2{:src=>"/images/wait30trans.gif", :id=>"circle_upload", :class=>"circle", :alt=>"processing"}
+ :javascript
+ $(document).ready(function() {
+ var aClient = new HttpClient();
+ aClient.post('//#{ENV['VIRTUAL_HOST']}/predict', 'models=#{@models}&batchfile=#{@filename}', function(res1) {
+ var contentType = res1.getResponseHeader('content-type');
+ if (contentType == "application/json"){
+ var response = JSON.parse(res1.responseText);
+ } else {
+ var response = res1.responseText;
+ }
+ if (res1.status == 200 && response['models'] && response['dataset_id']){
+ aClient.post('//#{ENV['VIRTUAL_HOST']}/predict', 'models='+response['models']+'&dataset_id='+response['dataset_id'], function(res2) {
+ $("body").html(res2.responseText);
+ });
+ }
+ if (res1.status == 200 && contentType == "text/html"){
+ $("body").html(response);
+ }
+ });
+ });