merge with origin/development

11 files changed, 217 insertions, 120 deletions

diff --git a/application.rb b/application.rb
index 14194ab..24b7a56 100644
--- a/application.rb
+++ b/application.rb
@@ -56,7 +56,7 @@ helpers do
 end
 
 before do
-    if !logged_in and !( env['REQUEST_URI'] =~ /\/login$/ and env['REQUEST_METHOD'] == "POST" ) #or !AA_SERVER
+    if !logged_in and !( env['REQUEST_URI'] =~ /\/login$/ and env['REQUEST_METHOD'] == "POST" ) or !AA_SERVER
       login("guest","guest")
     end
 end
@@ -175,9 +175,30 @@ get %r{/compound/(.*)} do |inchi|
   OpenTox::Compound.from_inchi(inchi).to_names.join(', ')
 end
 
+get '/echa' do
+  @endpoints = OpenTox::Ontology::Echa.endpoints
+  haml :echa
+end
+
+post '/ambit' do
+  session[:echa] = params[:endpoint]
+  @datasets  = OpenTox::Ontology::Echa.datasets(params[:endpoint])
+  haml :ambit
+end
+
+post '/feature' do
+  session[:dataset] = params[:dataset]
+  @features = []
+  OpenTox::Dataset.new(params[:dataset]).load_features.each do |uri,metadata|
+    @features << OpenTox::Feature.find(uri) if metadata[OWL.sameAs].match(/#{session[:echa]}/)
+  end
+  haml :feature
+end
+
 post '/models' do # create a new model
-  unless params[:file] and params[:file][:tempfile] #params[:endpoint] and 
-    flash[:notice] = "Please upload a Excel or CSV file."
+
+  unless (params[:dataset] and params[:prediction_feature]) or (params[:file] and params[:file][:tempfile]) #params[:endpoint] and 
+    flash[:notice] = "Please upload a Excel or CSV file or select an AMBIT dataset."
     redirect url_for('/create')
   end
 
@@ -186,69 +207,88 @@ post '/models' do # create a new model
     flash[:notice] = "Please login to create a new model."
     redirect url_for('/create')
   end
+
   subjectid = session[:subjectid] ? session[:subjectid] : nil
-  @model = ToxCreateModel.create(:name => params[:file][:filename].sub(/\..*$/,""), :subjectid => subjectid)
+
+  if params[:dataset] and params[:prediction_feature]
+    @dataset = OpenTox::Dataset.new(params[:dataset],subjectid)
+    name = @dataset.load_metadata[DC.title]
+    @prediction_feature = OpenTox::Feature.find params[:prediction_feature], subjectid
+    @dataset.load_compounds
+  elsif params[:file][:filename]
+    name = params[:file][:filename].sub(/\..*$/,"")
+  end
+
+  @model = ToxCreateModel.create(:name => name, :subjectid => subjectid)
   @model.update :web_uri => url_for("/model/#{@model.id}", :full), :warnings => ""
   task = OpenTox::Task.create("Uploading dataset and creating lazar model",url_for("/models",:full)) do |task|
 
     task.progress(5)
     @model.update :status => "Uploading and saving dataset", :task_uri => task.uri
-    begin
-      @dataset = OpenTox::Dataset.create(nil, subjectid)
-      # check format by extension - not all browsers provide correct content-type]) 
-      case File.extname(params[:file][:filename])
-      when ".csv"
-        csv = params[:file][:tempfile].read
-        @dataset.load_csv(csv, subjectid)
-      when ".xls", ".xlsx"
-        excel_file = params[:file][:tempfile].path + File.extname(params[:file][:filename])
-        File.rename(params[:file][:tempfile].path, excel_file) # add extension, spreadsheet does not read files without extensions
-        @dataset.load_spreadsheet(Excel.new excel_file, subjectid)
+
+    unless params[:dataset] and params[:prediction_feature]
+      begin
+        @dataset = OpenTox::Dataset.create(nil, subjectid)
+        # check format by extension - not all browsers provide correct content-type]) 
+        case File.extname(params[:file][:filename])
+        when ".csv"
+          csv = params[:file][:tempfile].read
+          @dataset.load_csv(csv, subjectid)
+        when ".xls", ".xlsx"
+          excel_file = params[:file][:tempfile].path + File.extname(params[:file][:filename])
+          File.rename(params[:file][:tempfile].path, excel_file) # add extension, spreadsheet does not read files without extensions
+          @dataset.load_spreadsheet(Excel.new excel_file, subjectid)
+          if @dataset.metadata[OT.Errors]
+            error "Incorrect file format. Please follow the instructions for #{link_to "Excel", "/excel_format"} or #{link_to "CSV", "/csv_format"} formats."
+          end
+        else
+          error "#{params[:file][:filename]} has a unsupported file type."
+        end
+        @dataset.save(subjectid)
+      rescue => e
+        error "Dataset creation failed with #{e.message}"
+      end
+      if @dataset.features.keys.size != 1
+        error "More than one feature in dataset #{params[:file][:filename]}. Please delete irrelvant columns and try again."
       else
-        error "#{params[:file][:filename]} has a unsupported file type."
+        @prediction_feature = OpenTox::Feature.find(@dataset.features.keys.first,subjectid)
       end
-    rescue => e
-      error "Dataset creation failed with #{e.message}"
     end
-    @dataset.save(subjectid)
+
     task.progress(10)
     if @dataset.compounds.size < 10
       error "Too few compounds to create a prediction model. Did you provide compounds in SMILES format and classification activities as described in the #{link_to "instructions", "/excel_format"}? As a rule of thumb you will need at least 100 training compounds for nongeneric datasets. A lower number could be sufficient for congeneric datasets."
     end
-    if @dataset.features.keys.size != 1
-      error "More than one feature in dataset #{params[:file][:filename]}. Please delete irrelvant columns and try again."
-    end
-    if @dataset.metadata[OT.Errors]
-      error "Incorrect file format. Please follow the instructions for #{link_to "Excel", "/excel_format"} or #{link_to "CSV", "/csv_format"} formats."
-    end
     @model.update :training_dataset => @dataset.uri, :nr_compounds => @dataset.compounds.size, :status => "Creating prediction model"
-    @model.update :warnings => @dataset.metadata[OT.Warnings] unless @dataset.metadata[OT.Warnings].empty?
+    @model.update :warnings => @dataset.metadata[OT.Warnings] unless @dataset.metadata[OT.Warnings] and @dataset.metadata[OT.Warnings].empty?
     task.progress(15)
     begin
-      lazar = OpenTox::Model::Lazar.create({:dataset_uri => @dataset.uri, :subjectid => subjectid})
+      lazar = OpenTox::Model::Lazar.create(:dataset_uri => @dataset.uri, :prediction_feature => @prediction_feature.uri, :subjectid => subjectid)
     rescue => e
-      error "Model creation failed with '#{e.message}'. Please check if the input file is in a valid #{link_to "Excel", "/excel_format"} or #{link_to "CSV", "/csv_format"} format."
+      error "Model creation failed with '#{e.message}'."# Please check if the input file is in a valid #{link_to "Excel", "/excel_format"} or #{link_to "CSV", "/csv_format"} format."
     end
     task.progress(25)
+=begin
     type = "unknown"
-    case lazar.metadata[OT.isA]
-    when /Classification/
+    if lazar.metadata[RDF.type].grep(/Classification/)
       type = "classification"
-    when /Regression/
+    elsif lazar.metadata[RDF.type].grep(/Regression/)
       type = "regression"
     end
-    @model.update :type => type, :feature_dataset => lazar.metadata[OT.featureDataset], :uri => lazar.uri
+=end
+    @model.update :type => @prediction_feature.feature_type, :feature_dataset => lazar.metadata[OT.featureDataset], :uri => lazar.uri
 
-    unless url_for("",:full).match(/localhost/)
+    if CONFIG[:services]["opentox-validation"]
       @model.update :status => "Validating model"
       begin
-        validation = OpenTox::Crossvalidation.create(
-          {:algorithm_uri => lazar.metadata[OT.algorithm],
-          :dataset_uri => lazar.parameter("dataset_uri"),
-          :subjectid => subjectid,
-          :prediction_feature => lazar.parameter("prediction_feature"),
-          :algorithm_params => "feature_generation_uri=#{lazar.parameter("feature_generation_uri")}"},
-         nil, OpenTox::SubTask.new(task,25,80))
+        validation = OpenTox::Crossvalidation.create( {
+            :algorithm_uri => lazar.metadata[OT.algorithm],
+            :dataset_uri => lazar.parameter("dataset_uri"),
+            :subjectid => subjectid,
+            :prediction_feature => lazar.parameter("prediction_feature"),
+            :algorithm_params => "feature_generation_uri=#{lazar.parameter("feature_generation_uri")}" },
+            nil, OpenTox::SubTask.new(task,25,80))
+
         @model.update(:validation_uri => validation.uri)
         LOGGER.debug "Validation URI: #{@model.validation_uri}"
 
@@ -273,14 +313,10 @@ post '/models' do # create a new model
         @model.update :warnings => @model.warnings + "\nModel validation failed with #{e.message}.", :status => "Error", :error_messages => e.message
       end
       
+    else
+      @model.update(:status => "Completed") #, :warnings => @model.warnings + "\nValidation service cannot be accessed from localhost.")
+      task.progress(100)
     end
-
-
-    #@model.warnings += "<p>Incorrect Smiles structures (ignored):</p>" + parser.smiles_errors.join("<br/>") unless parser.smiles_errors.empty?
-    #@model.warnings += "<p>Irregular activities (ignored):</p>" + parser.activity_errors.join("<br/>") unless parser.activity_errors.empty?
-    #duplicate_warnings = ''
-    #parser.duplicates.each {|inchi,lines| duplicate_warnings += "<p>#{lines.join('<br/>')}</p>" if lines.size > 1 }
-    #@model.warnings += "<p>Duplicated structures (all structures/activities used for model building, please  make sure, that the results were obtained from <em>independent</em> experiments):</p>" + duplicate_warnings unless duplicate_warnings.empty?
     lazar.uri
   end
   @model.update :task_uri => task.uri
diff --git a/helper.rb b/helper.rb
index 7d7a856..2fe8785 100644
--- a/helper.rb
+++ b/helper.rb
@@ -3,7 +3,7 @@ helpers do
   def login(username, password)
     logout
     session[:subjectid] = OpenTox::Authorization.authenticate(username, password)
-    LOGGER.debug "ToxCreate login user #{username} with subjectid: " + session[:subjectid].to_s
+    #LOGGER.debug "ToxCreate login user #{username} with subjectid: " + session[:subjectid].to_s
     if session[:subjectid] != nil
       session[:username] = username
       return true
@@ -53,8 +53,7 @@ helpers do
 
   def sort(descriptors)
     features = {:activating => [], :deactivating => []}
-
-    descriptors.each { |d| LOGGER.debug d.inspect; features[d[OT.effect].to_sym] << {:smarts => d[OT.smarts],:p_value => d[OT.pValue]} }
+    descriptors.each { |d| features[d[OT.effect].to_sym] << {:smarts => d[OT.smarts],:p_value => d[OT.pValue]} }
     features
   end
 
diff --git a/views/ambit.haml b/views/ambit.haml
new file mode 100644
index 0000000..685c0dc
--- /dev/null
+++ b/views/ambit.haml
@@ -0,0 +1,15 @@
+%b Select:
+= session[:echa].split('#').last
+= "-&gt;"
+%b Dataset 
+%em= "-&gt; Prediction Feature"
+
+%form{ :action => url_for('/feature'), :method => "post" }
+  %input{:type => 'hidden', :name => 'subjectid', :id => 'subjectid', :value => session[:subjectid]}
+
+  - @datasets.each do |dataset|
+    %br
+      %input{:type=>'radio', :name => "dataset", :value => dataset} #{OpenTox::Dataset.new(dataset).load_metadata[DC.title]}
+
+  %p
+    %input{:type => 'submit', :value => 'Next'}
diff --git a/views/compound_image.haml b/views/compound_image.haml
index 18944dc..25e0dbf 100644
--- a/views/compound_image.haml
+++ b/views/compound_image.haml
@@ -1,2 +1 @@
 %img{:src => compound.matching_smarts_image_uri(features[:activating].collect{|f| f[:smarts]},features[:deactivating].collect{|f| f[:smarts]}), :alt => compound.to_smiles}
-
diff --git a/views/create.haml b/views/create.haml
index 2149f3d..48669f4 100644
--- a/views/create.haml
+++ b/views/create.haml
@@ -17,18 +17,24 @@
     = link_to "instructions for creating training datasets", '/help'
     before submitting.
 
+
   %form{ :action => url_for('/models'), :method => "post", :enctype => "multipart/form-data" }
     %input{:type => 'hidden', :name => 'subjectid', :id => 'subjectid', :value => session[:subjectid]}
     %fieldset
       %label{:for => 'file'}
-        Select training data in
+        Upload training data in
         = link_to "Excel", '/help'
         or
         = link_to "CSV", '/help'
         format:
       %input{:type => 'file', :name => 'file', :id => 'file', :size => '41'}
     %input{ :type => "submit", :value => "Create model"}
-    = link_to "Cancel", '/create'
+    =# link_to "Cancel", '/create'
+
+  %p
+    or
+    %a{:href => "echa"} Select a AMBIT dataset
+    (experimental)
 
   -# explanations
   = haml :lazar_description, :layout => false
diff --git a/views/echa.haml b/views/echa.haml
new file mode 100644
index 0000000..1cabfa0
--- /dev/null
+++ b/views/echa.haml
@@ -0,0 +1,13 @@
+%b Select:
+%b ECHA Endpoint
+%em= "-&gt; Dataset -&gt; Prediction Feature"
+
+%form{ :action => url_for('/ambit'), :method => "post" }
+  %input{:type => 'hidden', :name => 'subjectid', :id => 'subjectid', :value => session[:subjectid]}
+
+  - @endpoints.each do |endpoint|
+    %br
+      %input{:type=>'radio', :name => "endpoint", :value => endpoint} #{endpoint.split('#').last}
+
+  %p
+    %input{:type => 'submit', :value => "Next"}
diff --git a/views/feature.haml b/views/feature.haml
new file mode 100644
index 0000000..c5b8293
--- /dev/null
+++ b/views/feature.haml
@@ -0,0 +1,26 @@
+%b Select:
+= session[:echa].split('#').last
+= "-&gt;"
+= OpenTox::Dataset.new(session[:dataset]).load_metadata[DC.title]
+= "-&gt;"
+%b Prediction feature
+
+%form{ :action => url_for('/models'), :method => "post" }
+
+  %input{:type => 'hidden', :name => 'subjectid', :id => 'subjectid', :value => session[:subjectid]}
+  %input{:type => 'hidden', :name => 'dataset', :id => 'dataset', :value => session[:dataset]}
+
+  - @features.each do |feature|
+    -# type = "unknown"
+    -# if feature.metadata[RDF.type].include?(OT.NominalFeature)
+      - type = "classification"
+    -# elsif feature.metadata[RDF.type].include?(OT.NumericFeature)
+      - type = "regression"
+
+    %br
+      %input{:type=>'radio', :name => "prediction_feature", :value => feature.uri} #{feature.metadata[DC.title]} (#{feature.feature_type}) 
+    %br
+      =# feature.inspect
+
+  %p
+    %input{:type => 'submit', :value => "Create model"}
diff --git a/views/layout.haml b/views/layout.haml
index c849dec..174db9f 100644
--- a/views/layout.haml
+++ b/views/layout.haml
@@ -17,7 +17,7 @@
       Create and evaluate models to predict toxicity
     .index
       %ul
-        %li{:class => ("selected" if /\/create/ =~ request.path )}
+        %li{:class => ("selected" if /\/create|echa|ambit|feature/ =~ request.path )}
           = link_to "Create", "/create"
         %li{:class => ("selected" if /models/ =~ request.path )}
           = link_to "Inspect", "/models"
@@ -49,5 +49,5 @@
     .footer
       &copy;
       %a{:href => 'http://www.in-silico.ch', :rel => "external"} in silico toxicology
-      2009-2010, powered by
+      2009-2011, powered by
       %a{:href => 'http://www.opentox.org', :rel => "external"} <span style="color:#5D308A;font-family:arial,sans-serif,helvetica;letter-spacing:-1px;">Open</span><span style="color:#000;font-family:arial,sans-serif,helvetica;font-weight:bold;letter-spacing:-1px;">Tox</span>
diff --git a/views/lazar.haml b/views/lazar.haml
index deb0c18..6ae6b20 100644
--- a/views/lazar.haml
+++ b/views/lazar.haml
@@ -1,59 +1,59 @@
-%p= link_to "New prediction", "/predict"

-.lazar-predictions

-

-  -# explanations

-  = haml :lazar_algorithm, :layout => false

-  = haml :confidence, :layout => false

-  = haml :similarity, :layout => false

-  = haml :significant_fragments, :layout => false

-  = haml :training_data, :layout => false

-

-  %a{:name => "prediction"}

-  %table

-    %thead

-      %tr

-        %th= @prediction.title

-        %th= toggle_link("#lazar_algorithm","Prediction")

-        %th= toggle_link("#confidence","Confidence")

-        %th Supporting information

-

-      %tr

-        -# %td

-          %img{:src => @compound.to_image_uri, :alt => @compound.to_smiles}

-        %td.image= compound_image(@compound,@prediction.descriptors(@compound))

-        %td= activity_markup(@prediction.value(@compound))

-        %td= sprintf('%.03g', @prediction.confidence(@compound)) 

-        -#%td= @prediction.confidence(@compound)

-        %td

-          %ul

-            %li

-              %a{:href => "#prediction", :id => "show_names"} Names and synonyms

-              :javascript

-                $("a#show_names").click(function () {

-                  $("#compound_names").load("#{File.join("/compound",@compound.inchi)}");  

-                  $("tr#names").toggle();  

-                });

-            %li= toggle_link("#fragments","Significant fragments")

-            -# This does not work, ask nina/vedrin

-            -# %li

-              %a{:href => "http://ambit.uni-plovdiv.bg:8080/ambit2/query/structure/?search=#{@compound.smiles}", :rel => "external"} Ambit data

-            -# %li

-              %a{:href => "http://www.ncbi.nlm.nih.gov/sites/entrez?cmd=PureSearch&db=pccompound&term=#{URI.encode('"'+@compound.inchi+'"[InChI]')}", :rel => "external"}  PubChem data

+%p= link_to "New prediction", "/predict"
+.lazar-predictions
+
+  -# explanations
+  = haml :lazar_algorithm, :layout => false
+  = haml :confidence, :layout => false
+  = haml :similarity, :layout => false
+  = haml :significant_fragments, :layout => false
+  = haml :training_data, :layout => false
+
+  %a{:name => "prediction"}
+  %table
+    %thead
+      %tr
+        %th= @prediction.title
+        %th= toggle_link("#lazar_algorithm","Prediction")
+        %th= toggle_link("#confidence","Confidence")
+        %th Supporting information
+
+      %tr
+        -# %td
+          %img{:src => @compound.to_image_uri, :alt => @compound.to_smiles}
+        %td.image= compound_image(@compound,@prediction.descriptors(@compound))
+        %td= activity_markup(@prediction.value(@compound))
+        %td= sprintf('%.03g', @prediction.confidence(@compound)) 
+        -#%td= @prediction.confidence(@compound)
+        %td
+          %ul
+            %li
+              %a{:href => "#prediction", :id => "show_names"} Names and synonyms
+              :javascript
+                $("a#show_names").click(function () {
+                  $("#compound_names").load("#{File.join("/compound",@compound.inchi)}");  
+                  $("tr#names").toggle();  
+                });
+            %li= toggle_link("#fragments","Significant fragments")
+            -# This does not work, ask nina/vedrin
+            -# %li
+              %a{:href => "http://ambit.uni-plovdiv.bg:8080/ambit2/query/structure/?search=#{@compound.smiles}", :rel => "external"} Ambit data
+            -# %li
+              %a{:href => "http://www.ncbi.nlm.nih.gov/sites/entrez?cmd=PureSearch&db=pccompound&term=#{URI.encode('"'+@compound.inchi+'"[InChI]')}", :rel => "external"}  PubChem data
                 (external)
-            -# %li

-              %a{:href => "http://chem.sis.nlm.nih.gov/chemidplus/direct.jsp?result=advanced&inchi=#{URI.encode @compound.inchi}", :rel => "external"} ToxNet data

-            -#http://chem.sis.nlm.nih.gov/chemidplus/direct.jsp?result=advanced&regno=000143157

-

-      %tr#names{ :style => "display: none;" }

-        %td{:colspan => '4'}

-          %a{:name => 'names'}

-          = hide_link('#names')

-          #compound_names

-      %tr#fragments{ :style => "display: none;" }

-        %td{:colspan => '4'}

-          = hide_link('#fragments')

-          = haml :feature_table, :locals => {:features => sort(@prediction.descriptors(@compound))}, :layout => false

-

-    %tbody#neighbors

-      = haml :neighbors, :locals => {:neighbors => @prediction.neighbors(@compound), :page => @page},  :layout => false

-

+            -# %li
+              %a{:href => "http://chem.sis.nlm.nih.gov/chemidplus/direct.jsp?result=advanced&inchi=#{URI.encode @compound.inchi}", :rel => "external"} ToxNet data
+            -#http://chem.sis.nlm.nih.gov/chemidplus/direct.jsp?result=advanced&regno=000143157
+
+      %tr#names{ :style => "display: none;" }
+        %td{:colspan => '4'}
+          %a{:name => 'names'}
+          = hide_link('#names')
+          #compound_names
+      %tr#fragments{ :style => "display: none;" }
+        %td{:colspan => '4'}
+          = hide_link('#fragments')
+          = haml :feature_table, :locals => {:features => sort(@prediction.descriptors(@compound))}, :layout => false
+
+    %tbody#neighbors
+      = haml :neighbors, :locals => {:neighbors => @prediction.neighbors(@compound), :page => @page},  :layout => false
+
diff --git a/views/model.haml b/views/model.haml
index 4933cdb..b4b3c59 100644
--- a/views/model.haml
+++ b/views/model.haml
@@ -58,12 +58,15 @@
       - if model.training_dataset
         %dt Training dataset:
         %dd
-          %a{:href => "#{model.training_dataset}.xls#{subjectstring}"} Excel sheet
-          ,
-          -#%a{:href => "#{model.training_dataset}.rdf"} RDF/XML
-          -#%em (experts) ,
-          %a{:href => "#{model.training_dataset}.yaml#{subjectstring}" } YAML
-          %em  (experts)
+          - if model.training_dataset.match(/ambit/i)
+            %a{:href => "#{model.training_dataset}#{subjectstring}", :rel => "external"} Ambit database
+          - else
+            %a{:href => "#{model.training_dataset}.xls#{subjectstring}"} Excel sheet
+            ,
+            -#%a{:href => "#{model.training_dataset}.rdf"} RDF/XML
+            -#%em (experts) ,
+            %a{:href => "#{model.training_dataset}.yaml#{subjectstring}" } YAML
+            %em  (experts)
       - if model.feature_dataset
         %dt Feature dataset:
         %dd
diff --git a/views/neighbors.haml b/views/neighbors.haml
index b507da0..2df9230 100644
--- a/views/neighbors.haml
+++ b/views/neighbors.haml
@@ -14,7 +14,7 @@
   - compound = OpenTox::Compound.new(neighbor[OT.compound])
   %tr
     %td.image= compound_image(compound,@prediction.descriptors(compound))
-    %td= activity_markup(neighbor[OT.activity])
+    %td= activity_markup(neighbor[OT.measuredActivity])
     %td= sprintf('%.03g', neighbor[OT.similarity])
     %td
       %ul