opentox-ruby-api-wrapper renamed to opentox-ruby

15 files changed, 376 insertions, 382 deletions

diff --git a/application.rb b/application.rb
index 6b8de41..fe9b8b4 100644
--- a/application.rb
+++ b/application.rb
@@ -1,252 +1,350 @@
 ['rubygems', "haml", "sass", "rack-flash"].each do |lib|
-	require lib
+  require lib
 end
-gem "opentox-ruby-api-wrapper", "= 1.6.6"
-require 'opentox-ruby-api-wrapper'
+gem "opentox-ruby", "~> 0"
+require 'opentox-ruby'
 gem 'sinatra-static-assets'
 require 'sinatra/static_assets'
 require 'ftools'
 require File.join(File.dirname(__FILE__),'model.rb')
 require File.join(File.dirname(__FILE__),'helper.rb')
-require File.join(File.dirname(__FILE__),'parser.rb')
+#require File.join(File.dirname(__FILE__),'parser.rb')
 
 use Rack::Flash
 set :sessions, true
 
-get '/?' do
-	redirect url_for('/create')
+helpers do 
+
+  def error(message)
+    @model.error_messages = message
+    LOGGER.error message
+    @model.status = "Error"
+    @model.save
+    #@dataset.delete
+    flash[:notice] = message
+    redirect url_for('/create')
+  end
+
 end
 
-get '/models/?' do
-	@models = ToxCreateModel.all(:order => [ :created_at.desc ])
-	@models.each { |model| model.process }
-	haml :models
+get '/?' do
+  redirect url_for('/create')
 end
 
-delete '/model/:id/?' do
-	model = ToxCreateModel.get(params[:id])
-	begin
-		RestClient.delete model.uri if model.uri
-		RestClient.delete model.task_uri if model.task_uri
-    model.destroy!
-    flash[:notice] = "#{model.name} model deleted."
-	rescue
-	  flash[:notice] = "#{model.name} model delete error."
-	end
-	redirect url_for('/models')
+get '/models/?' do
+  @models = ToxCreateModel.all(:order => [ :created_at.desc ])
+  #@models.each { |model| model.process }
+  haml :models
 end
 
 get '/model/:id/status/?' do
   response['Content-Type'] = 'text/plain'
-	model = ToxCreateModel.get(params[:id])
-	begin
-		haml :model_status, :locals=>{:model=>model}, :layout => false
-	rescue
+  model = ToxCreateModel.get(params[:id])
+  begin
+    haml :model_status, :locals=>{:model=>model}, :layout => false
+  rescue
     return "unavailable"
-	end
+  end
 end
 
 get '/model/:id/:view/?' do
   response['Content-Type'] = 'text/plain'
-	model = ToxCreateModel.get(params[:id])
+  model = ToxCreateModel.get(params[:id])
 
   begin
-    model.process
-    model.save
+    #model.process
+    #model.save
     case params[:view]
       when "model"
-		    haml :model, :locals=>{:model=>model}, :layout => false
-		  when /validation/
+        haml :model, :locals=>{:model=>model}, :layout => false
+      when /validation/
         haml :validation, :locals=>{:model=>model}, :layout => false
-		  else
-				return "unable to render model: id #{params[:id]}, view #{params[:view]}"
-		end
-	rescue
+      else
+        return "unable to render model: id #{params[:id]}, view #{params[:view]}"
+    end
+  rescue
     return "unable to render model: id #{params[:id]}, view #{params[:view]}"
-	end
+  end
 end
 
 get '/predict/?' do 
-	@models = ToxCreateModel.all(:order => [ :created_at.desc ])
-	@models = @models.collect{|m| m if m.status == 'Completed'}.compact
-	haml :predict
+  @models = ToxCreateModel.all(:order => [ :created_at.desc ])
+  @models = @models.collect{|m| m if m.status == 'Completed'}.compact
+  haml :predict
 end
 
 get '/create' do
-	haml :create
+  haml :create
 end
 
 get '/help' do
-	haml :help
+  haml :help
 end
 
 get "/confidence" do
-	haml :confidence
+  haml :confidence
+end
+
+# proxy to get data from compound service
+# (jQuery load does not work with external URIs)
+get %r{/compound/(.*)} do |inchi|
+  inchi = URI.unescape request.env['REQUEST_URI'].sub(/^\//,'').sub(/.*compound\//,'')
+  OpenTox::Compound.from_inchi(inchi).to_names.join(', ')
 end
 
-post '/upload' do # create a new model
-
-	if params[:endpoint] == ''
-		flash[:notice] = "Please enter an endpoint name."
-		redirect url_for('/create')
-	end
-	unless params[:endpoint] and params[:file] and params[:file][:tempfile]
-		flash[:notice] = "Please enter an endpoint name and upload a Excel or CSV file."
-		redirect url_for('/create')
-	end
-
-	@model = ToxCreateModel.new
-	@model.name = params[:endpoint]
-	feature_uri = url_for("/feature#"+URI.encode(params[:endpoint]), :full)
-	parser = Parser.new params[:file], feature_uri
-
-	unless parser.format_errors.empty?
-		flash[:notice] = "Incorrect file format. Please follow the instructions for #{link_to "Excel", "/excel_format"} or #{link_to "CSV", "/csv_format"} formats."
-	end
-
-	if parser.dataset.compounds.empty?
-		flash[:notice] = "Dataset #{params[:file][:filename]} is empty."
-		redirect url_for('/create')
-	end
-
-	begin
-		@model.task_uri = OpenTox::Algorithm::Lazar.create_model(:dataset_uri => parser.dataset_uri, :prediction_feature => feature_uri)
-	rescue
-		flash[:notice] = "Model creation failed. Please check if the input file is in a valid #{link_to "Excel", "/excel_format"} or #{link_to "CSV", "/csv_format"} format."
-		redirect url_for('/create')
-	end
+post '/models' do # create a new model
 
-  begin
-    validation_task_uri = OpenTox::Validation.crossvalidation(
-      :algorithm_uri => OpenTox::Algorithm::Lazar.uri,
-      :dataset_uri => parser.dataset_uri,
-      :prediction_feature => feature_uri,
-      :algorithm_params => "feature_generation_uri=#{OpenTox::Algorithm::Fminer.uri}"
-    ).uri
-    LOGGER.debug "Validation task: " + validation_task_uri
-    @model.validation_task_uri = validation_task_uri
-	rescue
-		flash[:notice] = "Model validation failed."
+  unless params[:file] and params[:file][:tempfile] #params[:endpoint] and 
+    flash[:notice] = "Please upload a Excel or CSV file."
+    redirect url_for('/create')
   end
 
-=begin
-	if parser.nr_compounds < 10
-		flash[:notice] = "Too few compounds to create a prediction model. Did you provide compounds in SMILES format and classification activities as described in the #{link_to "instructions", "/excel_format"}? As a rule of thumb you will need at least 100 training compounds for nongeneric datasets. A lower number could be sufficient for congeneric datasets."
-		redirect url_for('/create')
-	end
-=end
+  @model = ToxCreateModel.create(:name => params[:file][:filename].sub(/\..*$/,""))
+  task = OpenTox::Task.create("Uploading dataset and creating lazar model",url_for("/models",:full)) do
+
+    @model.update :status => "Uploading and saving dataset"
+    begin
+      @dataset = OpenTox::Dataset.create
+      # check format by extension - not all browsers provide correct content-type]) 
+      case File.extname(params[:file][:filename])
+      when ".csv"
+        csv = params[:file][:tempfile].read
+        LOGGER.debug csv
+        @dataset.load_csv(csv)
+      when ".xls", ".xlsx"
+        @dataset.load_spreadsheet(Excel.new params[:file][:tempfile].path)
+      else
+        error "#{params[:file][:filename]} has a unsupported file type."
+      end
+    rescue => e
+      error "Dataset creation failed with #{e.message}"
+    end
+    @dataset.save
+    if @dataset.compounds.size < 10
+      error "Too few compounds to create a prediction model. Did you provide compounds in SMILES format and classification activities as described in the #{link_to "instructions", "/excel_format"}? As a rule of thumb you will need at least 100 training compounds for nongeneric datasets. A lower number could be sufficient for congeneric datasets."
+    end
+    if @dataset.features.keys.size != 1
+      error "More than one feature in dataset #{params[:file][:filename]}. Please delete irrelvant columns and try again."
+    end
+    if @dataset.metadata[OT.Errors]
+      error "Incorrect file format. Please follow the instructions for #{link_to "Excel", "/excel_format"} or #{link_to "CSV", "/csv_format"} formats."
+    end
+    @model.training_dataset = @dataset.uri
+    @model.nr_compounds = @dataset.compounds.size
+    @model.warnings = @dataset.metadata[OT.Warnings] unless @dataset.metadata[OT.Warnings].empty?
+    @model.save
 
-	@model.nr_compounds = parser.nr_compounds
-	@model.warnings = ''
+    @model.update :status => "Creating prediction model"
+    begin
+      lazar = OpenTox::Model::Lazar.create(:dataset_uri => @dataset.uri)
+    rescue => e
+      error "Model creation failed with '#{e.message}'. Please check if the input file is in a valid #{link_to "Excel", "/excel_format"} or #{link_to "CSV", "/csv_format"} format."
+    end
+    LOGGER.debug lazar.metadata.to_yaml
+    @model.feature_dataset = lazar.metadata[OT.featureDataset]
+    @model.uri = lazar.uri
+    case lazar.metadata[OT.isA]
+    when /Classification/
+      @model.type = "classification"
+    when /Regression/
+      @model.type = "regression"
+    else
+      @model.type = "unknown"
+    end
+    @model.save
 
-	@model.warnings += "<p>Incorrect Smiles structures (ignored):</p>" + parser.smiles_errors.join("<br/>") unless parser.smiles_errors.empty?
-	@model.warnings += "<p>Irregular activities (ignored):</p>" + parser.activity_errors.join("<br/>") unless parser.activity_errors.empty?
-	duplicate_warnings = ''
-	parser.duplicates.each {|inchi,lines| duplicate_warnings += "<p>#{lines.join('<br/>')}</p>" if lines.size > 1 }
-	@model.warnings += "<p>Duplicated structures (all structures/activities used for model building, please  make sure, that the results were obtained from <em>independent</em> experiments):</p>" + duplicate_warnings unless duplicate_warnings.empty?
-	@model.save
+    unless url_for("",:full).match(/localhost/)
+      @model.update :status => "Validating model"
+      begin
+        validation = OpenTox::Validation.create_crossvalidation(
+          :algorithm_uri => OpenTox::Algorithm::Lazar.uri,
+          :dataset_uri => lazar.parameter("dataset_uri"),
+          :prediction_feature => lazar.parameter("prediction_feature"),
+          :algorithm_params => "feature_generation_uri=#{lazar.parameter("feature_generation_uri")}"
+        )
+        @model.update(:validation_uri => validation.uri)
+        LOGGER.debug "Validation URI: #{@model.validation_uri}"
+      rescue => e
+        LOGGER.debug "Model validation failed with #{e.message}."
+        @model.warnings += "Model validation failed with #{e.message}."
+      end
 
-	flash[:notice] = "Model creation and validation started - this may last up to several hours depending on the number and size of the training compounds."
-	redirect url_for('/models')
+      # create summary
+      validation.summary(@model.type).each{|k,v| eval "@model.#{k.to_s} = v"}
+      @model.save
+      
+      @model.update :status => "Creating validation report"
+      begin
+        @model.update(:validation_report_uri => validation.create_report)
+      rescue => e
+        LOGGER.debug "Validation report generation failed with #{e.message}."
+        @model.warnings += "Validation report generation failed with #{e.message}."
+      end
 
-	# TODO: check for empty model
+      @model.update :status => "Creating QMRF report"
+      begin
+        @model.update(:validation_qmrf_report_uri => validation.create_qmrf_report)
+      rescue => e
+        LOGGER.debug "Validation QMRF report generation failed with #{e.message}."
+        @model.warnings += "Validation QMRF report generation failed with #{e.message}."
+      end
+    end
+
+
+
+    #@model.warnings += "<p>Incorrect Smiles structures (ignored):</p>" + parser.smiles_errors.join("<br/>") unless parser.smiles_errors.empty?
+    #@model.warnings += "<p>Irregular activities (ignored):</p>" + parser.activity_errors.join("<br/>") unless parser.activity_errors.empty?
+    #duplicate_warnings = ''
+    #parser.duplicates.each {|inchi,lines| duplicate_warnings += "<p>#{lines.join('<br/>')}</p>" if lines.size > 1 }
+    #@model.warnings += "<p>Duplicated structures (all structures/activities used for model building, please  make sure, that the results were obtained from <em>independent</em> experiments):</p>" + duplicate_warnings unless duplicate_warnings.empty?
+    @model.update :status => "Completed"
+    lazar.uri
+  end
+  @model.update(:task_uri => task.uri)
+  @model.save
+
+  flash[:notice] = "Model creation and validation started - this may last up to several hours depending on the number and size of the training compounds."
+  redirect url_for('/models')
+
+=begin
+=end
 end
 
 post '/predict/?' do # post chemical name to model
-	@identifier = params[:identifier]
-	unless params[:selection] and params[:identifier] != ''
-		flash[:notice] = "Please enter a compound identifier and select an endpoint from the list."
-		redirect url_for('/predict')
-	end
-	begin
-		@compound = OpenTox::Compound.new(:name => params[:identifier])
-	rescue
-		flash[:notice] = "Could not find a structure for '#{@identifier}'. Please try again."
-		redirect url_for('/predict')
-	end
-	@predictions = []
-	params[:selection].keys.each do |id|
-		model = ToxCreateModel.get(id.to_i)
-		model.process unless model.uri
-		prediction = nil
-		confidence = nil
-		title = nil
-		db_activities = []
-		#LOGGER.debug "curl -X POST -d 'compound_uri=#{@compound.uri}' -H 'Accept:application/x-yaml' #{model.uri}"
-		prediction = YAML.load(`curl -X POST -d 'compound_uri=#{@compound.uri}' -H 'Accept:application/x-yaml' #{model.uri}`)
-    #prediction = YAML.load(OpenTox::Model::Lazar.predict(params[:compound_uri],params[:model_uri]))
-		source = prediction.creator
-		if prediction.data[@compound.uri]
-			if source.to_s.match(/model/) # real prediction
-				prediction = prediction.data[@compound.uri].first.values.first
-				#LOGGER.debug prediction[File.join(@@config[:services]["opentox-model"],"lazar#classification")]
-				#LOGGER.debug prediction[File.join(@@config[:services]["opentox-model"],"lazar#confidence")]
-				if !prediction[File.join(@@config[:services]["opentox-model"],"lazar#classification")].nil?
-					@predictions << {
+  @identifier = params[:identifier]
+  unless params[:selection] and params[:identifier] != ''
+    flash[:notice] = "Please enter a compound identifier and select an endpoint from the list."
+    redirect url_for('/predict')
+  end
+  begin
+    @compound = OpenTox::Compound.from_name(params[:identifier])
+  rescue
+    flash[:notice] = "Could not find a structure for '#{@identifier}'. Please try again."
+    redirect url_for('/predict')
+  end
+  @predictions = []
+  params[:selection].keys.each do |id|
+    model = ToxCreateModel.get(id.to_i)
+    #model.process unless model.uri
+    prediction = nil
+    confidence = nil
+    title = nil
+    db_activities = []
+    lazar = OpenTox::Model::Lazar.new model.uri
+    prediction_dataset_uri = lazar.run(:compound_uri => @compound.uri)
+    prediction_dataset = OpenTox::LazarPrediction.find(prediction_dataset_uri)
+    if prediction_dataset.metadata[OT.hasSource].match(/dataset/)
+      @predictions << {
+        :title => model.name,
+        :measured_activities => prediction_dataset.measured_activities(@compound)
+      }
+    else
+      predicted_feature = prediction_dataset.metadata[OT.dependentVariables]
+      prediction = OpenTox::Feature.find(predicted_feature)
+      LOGGER.debug prediction.to_yaml
+      if prediction.metadata[OT.error]
+        @predictions << {
+          :title => model.name,
+          :error => prediction.metadata[OT.error]
+          }
+      else
+        @predictions << {
+          :title => model.name,
+          :model_uri => model.uri,
+          :prediction => prediction.metadata[OT.prediction],
+          :confidence => prediction.metadata[OT.confidence]
+          }
+      end
+    end
+    # TODO failed/unavailable predictions
+=begin
+    source = prediction.creator
+    if prediction.data[@compound.uri]
+      if source.to_s.match(/model/) # real prediction
+        prediction = prediction.data[@compound.uri].first.values.first
+        #LOGGER.debug prediction[File.join(CONFIG[:services]["opentox-model"],"lazar#classification")]
+        #LOGGER.debug prediction[File.join(CONFIG[:services]["opentox-model"],"lazar#confidence")]
+        if !prediction[File.join(CONFIG[:services]["opentox-model"],"lazar#classification")].nil?
+          @predictions << {
             :title => model.name,
             :model_uri => model.uri,
-            :prediction => prediction[File.join(@@config[:services]["opentox-model"],"lazar#classification")],
-            :confidence => prediction[File.join(@@config[:services]["opentox-model"],"lazar#confidence")]
+            :prediction => prediction[File.join(CONFIG[:services]["opentox-model"],"lazar#classification")],
+            :confidence => prediction[File.join(CONFIG[:services]["opentox-model"],"lazar#confidence")]
             }
-				elsif !prediction[File.join(@@config[:services]["opentox-model"],"lazar#regression")].nil?
-					@predictions << {
+        elsif !prediction[File.join(CONFIG[:services]["opentox-model"],"lazar#regression")].nil?
+          @predictions << {
             :title => model.name,
             :model_uri => model.uri,
-            :prediction => prediction[File.join(@@config[:services]["opentox-model"],"lazar#regression")],
-            :confidence => prediction[File.join(@@config[:services]["opentox-model"],"lazar#confidence")]
+            :prediction => prediction[File.join(CONFIG[:services]["opentox-model"],"lazar#regression")],
+            :confidence => prediction[File.join(CONFIG[:services]["opentox-model"],"lazar#confidence")]
             }
-				end
-			else # database value
-				prediction = prediction.data[@compound.uri].first.values
-				@predictions << {:title => model.name, :measured_activities => prediction}
-			end
-		else
-			@predictions << {:title => model.name, :prediction => "not available (not enough similar compounds in the training dataset)"}
-		end
-	end
-	LOGGER.debug @predictions.inspect
-
-	haml :prediction
+        end
+      else # database value
+        prediction = prediction.data[@compound.uri].first.values
+        @predictions << {:title => model.name, :measured_activities => prediction}
+      end
+    else
+      @predictions << {:title => model.name, :prediction => "not available (not enough similar compounds in the training dataset)"}
+    end
+=end
+  end
+  LOGGER.debug @predictions.inspect
+
+  haml :prediction
 end
 
 post "/lazar/?" do # get detailed prediction
   @page = 0
   @page = params[:page].to_i if params[:page]
   @model_uri = params[:model_uri]
-  @prediction = YAML.load(OpenTox::Model::Lazar.predict(params[:compound_uri],params[:model_uri]))
-  @compound = OpenTox::Compound.new(:uri => params[:compound_uri])
-  @title = @prediction.title
-  if @prediction.data[@compound.uri]
-    if @prediction.creator.to_s.match(/model/) # real prediction
-      p = @prediction.data[@compound.uri].first.values.first
-      if !p[File.join(@@config[:services]["opentox-model"],"lazar#classification")].nil?
-        feature = File.join(@@config[:services]["opentox-model"],"lazar#classification")
-      elsif !p[File.join(@@config[:services]["opentox-model"],"lazar#regression")].nil?
-        feature = File.join(@@config[:services]["opentox-model"],"lazar#regression")
-      end
-      @activity = p[feature]
-      @confidence = p[File.join(@@config[:services]["opentox-model"],"lazar#confidence")]
-      @neighbors = p[File.join(@@config[:services]["opentox-model"],"lazar#neighbors")]
-      @features = p[File.join(@@config[:services]["opentox-model"],"lazar#features")]
-    else # database value
-      @measured_activities = @prediction.data[@compound.uri].first.values
-    end
-  else
-    @activity = "not available (no similar compounds in the training dataset)"
-  end
+  lazar = OpenTox::Model::Lazar.new @model_uri
+  prediction_dataset_uri = lazar.run(:compound_uri => params[:compound_uri])
+  @prediction = OpenTox::LazarPrediction.find(prediction_dataset_uri)
+  @compound = OpenTox::Compound.new(params[:compound_uri])
+  #@title = prediction.metadata[DC.title]
+  # TODO dataset activity
+  #@activity = prediction.metadata[OT.prediction]
+  #@confidence = prediction.metadata[OT.confidence]
+  #@neighbors = []
+  #@features = []
+#  if @prediction.data[@compound.uri]
+#    if @prediction.creator.to_s.match(/model/) # real prediction
+#      p = @prediction.data[@compound.uri].first.values.first
+#      if !p[File.join(CONFIG[:services]["opentox-model"],"lazar#classification")].nil?
+#        feature = File.join(CONFIG[:services]["opentox-model"],"lazar#classification")
+#      elsif !p[File.join(CONFIG[:services]["opentox-model"],"lazar#regression")].nil?
+#        feature = File.join(CONFIG[:services]["opentox-model"],"lazar#regression")
+#      end
+#      @activity = p[feature]
+#      @confidence = p[File.join(CONFIG[:services]["opentox-model"],"lazar#confidence")]
+#      @neighbors = p[File.join(CONFIG[:services]["opentox-model"],"lazar#neighbors")]
+#      @features = p[File.join(CONFIG[:services]["opentox-model"],"lazar#features")]
+#    else # database value
+#      @measured_activities = @prediction.data[@compound.uri].first.values
+#    end
+#  else
+#    @activity = "not available (no similar compounds in the training dataset)"
+#  end
   haml :lazar
 end
 
-# proxy to get data from compound service
-# (jQuery load does not work with external URIs)
-get %r{/compound/(.*)} do |inchi|
-  OpenTox::Compound.new(:inchi => inchi).names.gsub(/\n/,', ')
+delete '/model/:id/?' do
+  model = ToxCreateModel.get(params[:id])
+  begin
+    RestClient.delete model.uri if model.uri
+    RestClient.delete model.task_uri if model.task_uri
+    model.destroy
+    flash[:notice] = "#{model.name} model deleted."
+  rescue
+    flash[:notice] = "#{model.name} model delete error."
+  end
+  redirect url_for('/models')
 end
 
 delete '/?' do
-  ToxCreateModel.auto_migrate!
-	response['Content-Type'] = 'text/plain'
-	"All Models deleted."
+  DataMapper.auto_migrate!
+  response['Content-Type'] = 'text/plain'
+  "All Models deleted."
 end
 
 # SASS stylesheet
diff --git a/helper.rb b/helper.rb
index d100c9e..d691103 100644
--- a/helper.rb
+++ b/helper.rb
@@ -12,8 +12,16 @@ helpers do
     haml :js_link, :locals => {:name => name, :destination => destination, :method => "toggle"}, :layout => false
   end
 
-  def compound_image(compound,features)
-    haml :compound_image, :locals => {:compound => compound, :features => features}, :layout => false
+  def sort(descriptors)
+    features = {:activating => [], :deactivating => []}
+
+    descriptors.each { |d| LOGGER.debug d.inspect; features[d[OT.effect].to_sym] << {:smarts => d[OT.smarts],:p_value => d[OT.pValue]} }
+    LOGGER.debug features.to_yaml
+    features
+  end
+
+  def compound_image(compound,descriptors)
+    haml :compound_image, :locals => {:compound => compound, :features => sort(descriptors)}, :layout => false
   end
   
   def activity_markup(activity)
diff --git a/model.rb b/model.rb
index da12f0b..9929ab0 100644
--- a/model.rb
+++ b/model.rb
@@ -5,19 +5,23 @@ class ToxCreateModel
 	property :id, Serial
 	property :name, String, :length => 255
 	property :warnings, Text, :length => 2**32-1 
+	property :error_messages, Text, :length => 2**32-1  # :errors interferes with datamapper validation
 	property :type, String
+  property :status, String, :length => 255
 	property :created_at, DateTime
 
 	property :task_uri, String, :length => 255
 	property :uri, String, :length => 255
 
-	property :validation_task_uri, String, :length => 255
+	property :training_dataset, String, :length => 255
+	property :feature_dataset, String, :length => 255
+	#property :validation_task_uri, String, :length => 255
 	property :validation_uri, String, :length => 255
 
-	property :validation_report_task_uri, String, :length => 255
+	#property :validation_report_task_uri, String, :length => 255
 	property :validation_report_uri, String, :length => 255
 
-	property :validation_qmrf_task_uri, String, :length => 255
+	#property :validation_qmrf_task_uri, String, :length => 255
 	property :validation_qmrf_uri, String, :length => 255
 
 	property :nr_compounds, Integer
@@ -34,13 +38,16 @@ class ToxCreateModel
 	property :root_mean_squared_error, Float
 	property :mean_absolute_error, Float
 
-	def status
+=begin
+def status
 		#begin
 			RestClient.get(File.join(@task_uri, 'hasStatus')).body
 		#rescue
 		#	"Service offline"
 		#end
 	end
+=end
+=begin
 
 	def validation_status
 		begin
@@ -92,29 +99,23 @@ class ToxCreateModel
 
   def process
 
+    LOGGER.debug self.to_yaml
+
     if @uri.nil? and status == "Completed"
-			update :uri => RestClient.get(File.join(@task_uri, 'resultURI')).body
-			lazar = YAML.load(RestClient.get(@uri, :accept => "application/x-yaml").body)
-			case lazar.dependentVariables
-			when /classification/
-				update :type => "classification"
-			when /regression/
-				update :type => "regression"
-			else
-				update :type => "unknown"
-			end
+			#update :uri => RestClient.get(File.join(@task_uri, 'resultURI')).body
+			#lazar = YAML.load(RestClient.get(@uri, :accept => "application/x-yaml").body)
 
 		elsif @validation_uri.nil? and validation_status == "Completed"
 			begin
 
-				update :validation_uri => RestClient.get(File.join(@validation_task_uri, 'resultURI')).body
-				LOGGER.debug "Validation URI: #{@validation_uri}"
+				#update :validation_uri => RestClient.get(File.join(@validation_task_uri, 'resultURI')).body
+				#LOGGER.debug "Validation URI: #{@validation_uri}"
 
-				update :validation_report_task_uri => RestClient.post(File.join(@@config[:services]["opentox-validation"],"/report/crossvalidation"), :validation_uris => @validation_uri).body
-				LOGGER.debug "Validation Report Task URI: #{@validation_report_task_uri}"
+				#update :validation_report_task_uri => RestClient.post(File.join(CONFIG[:services]["opentox-validation"],"/report/crossvalidation"), :validation_uris => @validation_uri).body
+				#LOGGER.debug "Validation Report Task URI: #{@validation_report_task_uri}"
 
-				update :validation_qmrf_task_uri => RestClient.post(File.join(@@config[:services]["opentox-validation"],"/reach_report/qmrf"), :model_uri => @uri).body
-				LOGGER.debug "QMRF Report Task URI: #{@validation_qmrf_task_uri}"
+				#update :validation_qmrf_task_uri => RestClient.post(File.join(CONFIG[:services]["opentox-validation"],"/reach_report/qmrf"), :model_uri => @uri).body
+				#LOGGER.debug "QMRF Report Task URI: #{@validation_qmrf_task_uri}"
 
         uri = File.join(@validation_uri, 'statistics')
         yaml = RestClient.get(uri).body
@@ -178,6 +179,7 @@ class ToxCreateModel
     end
 
   end
+=end
 
 end
 
diff --git a/parser.rb b/parser.rb
deleted file mode 100644
index 8468cea..0000000
--- a/parser.rb
+++ /dev/null
@@ -1,119 +0,0 @@
-require 'spreadsheet'
-require 'roo'
-class Parser
-
-  attr_accessor :dataset, :format_errors, :smiles_errors, :activity_errors, :duplicates, :nr_compounds, :dataset_uri
-
-  def initialize(file, endpoint_uri)
-
-		@file = file
-    @dataset = OpenTox::Dataset.new
-    @feature_uri = endpoint_uri
-		@dataset.features << endpoint_uri
-		@dataset.title = URI.decode(endpoint_uri.split(/#/).last)
-    @format_errors = ""
-    @smiles_errors = []
-    @activity_errors = []
-    @duplicates = {}
-    @nr_compounds = 0
-		@data = []
-		@activities = []
-		@type = "classification"
-
-		# check format by extension - not all browsers provide correct content-type]) 
-		case File.extname(@file[:filename])
-		when ".csv"
-			self.csv
-		when ".xls", ".xlsx"
-			self.excel
-		else
-			@format_errors = "#{@file[:filename]} is a unsupported file type."
-			return false
-		end
-
-		# create dataset
-		@data.each do |items|
-			@dataset.compounds << items[0]
-        @dataset.data[items[0]] = [] unless @dataset.data[items[0]]
-				case @type
-				when "classification"
-					case items[1].to_s
-					when TRUE_REGEXP
-						@dataset.data[items[0]] << {@feature_uri => true }
-					when FALSE_REGEXP
-						@dataset.data[items[0]] << {@feature_uri => false }
-					end
-				when "regression"
-					if items[1].to_f == 0
-						@activity_errors << "Row #{items[2]}: Zero values not allowed for regression datasets - entry ignored."
-					else
-						@dataset.data[items[0]] << {@feature_uri => items[1].to_f}
-					end
-				end
-		end
-		@dataset_uri = @dataset.save
-
-  end
-
-  def csv
-		row = 0
-    @file[:tempfile].each_line do |line|
-			row += 1
-      unless line.chomp.match(/^.+[,;].*$/) # check CSV format 
-				@format_errors = "#{@file[:filename]} is not a valid CSV file."
-        return false
-      end
-      items = line.chomp.gsub(/["']/,'').split(/\s*[,;]\s*/) # remove quotes
-			LOGGER.debug items.join(",")
-			input = validate(items[0], items[1], row) # smiles, activity
-			@data << input if input
-		end
-  end
-
-  def excel
-		excel = 'tmp/' + @file[:filename]
-		File.mv(@file[:tempfile].path,excel)
-		begin
-			if File.extname(@file[:filename]) == ".xlsx"    
-			 book = Excelx.new(excel)
-			else
-			 book = Excel.new(excel)
-			end
-			book.default_sheet = 0
-			1.upto(book.last_row) do |row|
-				input = validate( book.cell(row,1), book.cell(row,2), row ) # smiles, activity
-				@data << input if input
-			end
-			File.safe_unlink(@file[:tempfile])
-		rescue
-			@format_errors = "#{@file[:filename]} is not a valid Excel input file."
-			return false
-		end
-  end
-
-	def validate(smiles, act, row)
-		compound = OpenTox::Compound.new(:smiles => smiles)
-		if compound.nil? or compound.inchi.nil? or compound.inchi == ""
-			@smiles_errors << "Row #{row}: " + [smiles,act].join(", ") 
-			return false
-		end
-		unless numeric?(act) or classification?(act)
-			@activity_errors << "Row #{row}: " + [smiles,act].join(", ")
-			return false
-		end
-		@duplicates[compound.inchi] = [] unless @duplicates[compound.inchi]
-		@duplicates[compound.inchi] << "Row #{row}: " + [smiles, act].join(", ")
-		@type = "regression" unless classification?(act)
-		@nr_compounds += 1
-		[ compound.uri, act , row ]
-	end
-
-	def numeric?(object)
-		true if Float(object) rescue false
-	end
-
-	def classification?(object)
-		!object.to_s.strip.match(TRUE_REGEXP).nil? or !object.to_s.strip.match(FALSE_REGEXP).nil?
-	end
-
-end
diff --git a/public/javascripts/toxcreate.js b/public/javascripts/toxcreate.js
index d1ed490..500b685 100755
--- a/public/javascripts/toxcreate.js
+++ b/public/javascripts/toxcreate.js
@@ -85,7 +85,8 @@ $(function() {
           if(/^\d+$/.test(reload_id)) loadModel(reload_id, 'validation');
         };
     });
-    var validationCheck = setTimeout('checkValidation()',15000);
+    //var validationCheck = setTimeout('checkValidation()',15000);
+    var validationCheck = setTimeout('checkValidation()',5000);
   }
 });
 
diff --git a/views/compound_image.haml b/views/compound_image.haml
index c6f962a..18944dc 100644
--- a/views/compound_image.haml
+++ b/views/compound_image.haml
@@ -1,2 +1,2 @@
-%img{:src => compound.display_smarts_uri(features[:activating].collect{|f| f[:smarts]},@features[:deactivating].collect{|f| f[:smarts]}), :alt => compound.smiles}
+%img{:src => compound.matching_smarts_image_uri(features[:activating].collect{|f| f[:smarts]},features[:deactivating].collect{|f| f[:smarts]}), :alt => compound.to_smiles}
 
diff --git a/views/create.haml b/views/create.haml
index 20de401..6563494 100644
--- a/views/create.haml
+++ b/views/create.haml
@@ -18,13 +18,13 @@
     = link_to "instructions for creating training datasets", '/help'
     before submitting.
 
-  %form{ :action => url_for('/upload'), :method => "post", :enctype => "multipart/form-data" }
+  %form{ :action => url_for('/models'), :method => "post", :enctype => "multipart/form-data" }
     %fieldset
-      %label{:for => 'endpoint'} 1. Enter #{toggle_link("#endpoint_description","endpoint")} name and #{toggle_link("#unit","unit")} (for #{toggle_link("#regression","regression")}):
-      %input{:type => 'text', :name => 'endpoint', :id => 'endpoint', :size => '50'}
-      %br
+      -#%label{:for => 'endpoint'} 1. Enter #{toggle_link("#endpoint_description","endpoint")} name and #{toggle_link("#unit","unit")} (for #{toggle_link("#regression","regression")}):
+      -#%input{:type => 'text', :name => 'endpoint', :id => 'endpoint', :size => '50'}
+      -#%br
       %label{:for => 'file'}
-        2. Upload training data in
+        Select training dataset in 
         = link_to "Excel", '/help'
         or
         = link_to "CSV", '/help'
diff --git a/views/lazar.haml b/views/lazar.haml
index 59d5396..a46f82c 100644
--- a/views/lazar.haml
+++ b/views/lazar.haml
@@ -12,22 +12,25 @@
   %table
     %thead
       %tr
-        %th= @title.gsub(/_lazar_.*$/,' ').capitalize
+        %th= @prediction.title
         %th= toggle_link("#lazar_algorithm","Prediction")
         %th= toggle_link("#confidence","Confidence")
         %th Supporting information
 
       %tr
-        %td.image= compound_image(@compound,@features)
-        %td= activity_markup(@activity)
-        %td= sprintf('%.03g', @confidence.to_f.abs) if @confidence
+        -# %td
+          %img{:src => @compound.to_image_uri, :alt => @compound.to_smiles}
+        %td.image= compound_image(@compound,@prediction.descriptors(@compound))
+        %td= activity_markup(@prediction.value(@compound))
+        %td= sprintf('%.03g', @prediction.confidence(@compound)) 
+        -#%td= @prediction.confidence(@compound)
         %td
           %ul
             %li
               %a{:href => "#prediction", :id => "show_names"} Names and synonyms
               :javascript
                 $("a#show_names").click(function () {
-                  $("#compound_names").load("#{File.join("compound",@compound.inchi)}");  
+                  $("#compound_names").load("#{File.join("/compound",@compound.inchi)}");  
                   $("tr#names").toggle();  
                 });
             %li= toggle_link("#fragments","Significant fragments")
@@ -49,7 +52,7 @@
       %tr#fragments{ :style => "display: none;" }
         %td{:colspan => '4'}
           = hide_link('#fragments')
-          = haml :feature_table, :locals => {:features => @features}, :layout => false
+          = haml :feature_table, :locals => {:features => sort(@prediction.descriptors(@compound))}, :layout => false
 
     %tbody#neighbors
-      = haml :neighbors, :locals => {:neighbors => @neighbors, :page => @page},  :layout => :false
+      = haml :neighbors, :locals => {:neighbors => @prediction.neighbors(@compound), :page => @page},  :layout => :false
diff --git a/views/model.haml b/views/model.haml
index 0c8ea8f..d34079d 100644
--- a/views/model.haml
+++ b/views/model.haml
@@ -8,7 +8,6 @@
 %div{:id => "model_#{model.id}"}  
   %h2
     = model.name
-
   .model
     %dl
       %dt Status:
@@ -20,24 +19,25 @@
         )
       %dt Started:
       %dd= model.created_at.strftime("%m/%d/%Y - %I:%M:%S%p")
-      %dt Training compounds:
-      %dd= model.nr_compounds
-      %dt Warnings:
-      - if model.warnings == ''
-        %dd -
-      - else
+      - if model.nr_compounds
+        %dt Training compounds:
+        %dd= model.nr_compounds 
+      - if model.error_messages
+        %dt Errors:
+        %dd= model.error_messages
+      - if model.warnings
+        %dt Warnings:
         %a{:href => "#", :id => "show_model_#{model.id}_warnings"} show
         %dd{:id => "model_#{model.id}_warnings", :style => "display: none;"}= model.warnings
-  
-      - if model.status == 'Completed'
-        %dt Algorithm:
-        %dd
-          = toggle_link("#lazar_description","lazar")
+      %dt Algorithm:
+      %dd= toggle_link("#lazar_description","lazar")
+      - if model.type
         %dt Type:
         %dd= toggle_link("##{model.type}","#{model.type}")
-        %dt Descriptors:
-        %dd
-          %a{:href => 'http://www.maunz.de/libfminer2-bbrc-doc/'} Fminer backbone refinement classes
+      %dt Descriptors:
+      %dd
+        %a{:href => 'http://www.maunz.de/libfminer2-bbrc-doc/'} Fminer backbone refinement classes
+      - if model.training_dataset
         %dt Training dataset:
         %dd
           %a{:href => "#{model.training_dataset}.xls"} Excel sheet
@@ -46,18 +46,22 @@
           -#%em (experts) ,
           %a{:href => "#{model.training_dataset}.yaml"} YAML
           %em  (experts)
+      - if model.feature_dataset
         %dt Feature dataset:
         %dd
           -#%a{:href => "#{model.feature_dataset}.rdf"} RDF/XML
           -#,
+          %a{:href => "#{model.feature_dataset}.xls"} Excel sheet
+          ,
           %a{:href => "#{model.feature_dataset}.yaml"} YAML
-          %em  (experts, dataset too large for Excel)
+          %em  (experts)
+      - if model.uri
         %dt Model:
         %dd
           -#%a{:href => "#{model.uri}.rdf"} RDF/XML
           -#,
-          - unless model.validation_qmrf_uri.nil?
+          - if model.validation_qmrf_uri
             %a{:href => File.join(model.validation_qmrf_uri,"editor")} QMRF Editor, 
           %a{:href => "#{model.uri}.yaml"} YAML
           %em  (experts, models cannot be represented in Excel)
-        = haml :validation, :locals=>{:model=>model}, :layout => false     
+      = haml :validation, :locals=>{:model=>model}, :layout => false     
diff --git a/views/model_status.haml b/views/model_status.haml
index 1357d99..897e792 100644
--- a/views/model_status.haml
+++ b/views/model_status.haml
@@ -1,2 +1,2 @@
-= image_tag("/snake_transparent.gif") if model.status.match(/running|started|created/i)
+-#= image_tag("/snake_transparent.gif") if model.status.match(/running|started|created/i)
 = model.status
diff --git a/views/models.haml b/views/models.haml
index 6712e3d..17d9c8a 100644
--- a/views/models.haml
+++ b/views/models.haml
@@ -9,7 +9,8 @@
     if(reload_validation) setTimeout('checkValidation()',15000);
   });
 
-%p Get an overview about ToxCreate models. This page is refreshed every 15 seconds to update the model status.
+-# %p Get an overview about ToxCreate models. This page is refreshed every 15 seconds to update the model status.
+%p Get an overview about ToxCreate models. This page is refreshed every 5 seconds to update the model status.
 
 -# explanations
 = haml :lazar_description, :layout => false
diff --git a/views/neighbors.haml b/views/neighbors.haml
index 82fafd2..87fef13 100644
--- a/views/neighbors.haml
+++ b/views/neighbors.haml
@@ -9,15 +9,14 @@
 - first = 5*page
 - last = first+4
 - neighbor_id = 0
-- neighbors.sort{|a,b| b.last[:similarity] <=> a.last[:similarity]}[first..last].each do |uri,data|
+-# LOGGER.debug neighbors.to_yaml
+- neighbors.sort{|a,b| b[OT.similarity] <=> a[OT.similarity]}[first..last].each do |neighbor|
   - neighbor_id += 1
-  - compound = OpenTox::Compound.new(:uri => uri)
+  - compound = OpenTox::Compound.new(neighbor[OT.compound])
   %tr
-    %td.image= compound_image(compound,data[:features])
-    %td
-      - data[:activities].each do |act|
-        = activity_markup(act)
-    %td= sprintf('%.03g', data[:similarity])
+    %td.image= compound_image(compound,@prediction.descriptors(compound))
+    %td= activity_markup(neighbor[OT.activity])
+    %td= sprintf('%.03g', neighbor[OT.similarity])
     %td
       %ul
         %li
@@ -41,5 +40,5 @@
   %tr{:id => "fragments#{neighbor_id}", :style => "display: none;" }
     %td{:colspan => '4'}
       = hide_link("#fragments#{neighbor_id}")
-      = haml :feature_table, :locals => {:features => data[:features]}, :layout => false
+      = haml :feature_table, :locals => {:features => sort(@prediction.descriptors(compound))}, :layout => false
 
diff --git a/views/neighbors_navigation.haml b/views/neighbors_navigation.haml
index 43ec3ed..864a99f 100644
--- a/views/neighbors_navigation.haml
+++ b/views/neighbors_navigation.haml
@@ -7,9 +7,9 @@
 
     #prev= "prev" unless @page.to_i == 0
 
-    = "(#{5*@page+1}-#{5*@page+5}/#{@neighbors.size})" 
+    = "(#{5*@page+1}-#{5*@page+5}/#{@prediction.neighbors(@compound).size})" 
 
-    #next= "next" unless 5*@page.to_i+5 >= @neighbors.size
+    #next= "next" unless 5*@page.to_i+5 >= @prediction.neighbors(@compound).size
 
     :javascript
       $("#prev").click(function() {
diff --git a/views/prediction.haml b/views/prediction.haml
index 0140b5e..8498abc 100644
--- a/views/prediction.haml
+++ b/views/prediction.haml
@@ -7,8 +7,9 @@
         = @identifier
     %tr
       %td
-        %img{:src => @compound.image_uri, :alt => @compound.smiles}
+        %img{:src => @compound.to_image_uri, :alt => @compound.to_smiles}
       - @predictions.each do |p|
+        - LOGGER.debug p.to_yaml
         %td
           %b
             = p[:title] + ":"
@@ -25,8 +26,8 @@
                 });
               )
 
-          - elsif p[:prediction].to_s.match(/not available/)
-            = p[:prediction]
+          - elsif p[:error]
+            %br= p[:error]
           - else
             = activity_markup(p[:prediction])
             - if p[:confidence]
diff --git a/views/validation.haml b/views/validation.haml
index bb44058..5fc7371 100644
--- a/views/validation.haml
+++ b/views/validation.haml
@@ -1,18 +1,14 @@
-%dl{:id => "model_validation_#{model.id}"}
-  %dt
-    Validation:
-    %input{ :id => "model_validation_report_#{model.id}", :type => "hidden", :value => "#{model.validation_report_status}", :class => "model_validation_report" }
-  %dd
-    - if model.validation_uri
+- if model.validation_uri
+  %dl{:id => "model_validation_#{model.id}"}
+    %dt
+      Validation:
+      -# %input{ :id => "model_validation_report_#{model.id}", :type => "hidden", :value => "#{model.validation_report_status}", :class => "model_validation_report" }
+    %dd
       %dl
-        %dt
-          Detailed report:
-        %dd
-          - if model.validation_report_uri
+        - if model.validation_report_uri
+          %dt Detailed report:
+          %dd
             %a{:href => model.validation_report_uri, :target => "_blank"} show
-          - else
-            = image_tag("/snake_transparent.gif") if model.validation_report_status == "Running"
-            %a{:href => model.validation_report_task_uri} #{model.validation_report_status}
         %dt Number of predictions
         %dd= model.nr_predictions
         - case model.type
@@ -20,8 +16,8 @@
           = haml :classification_validation, :locals=>{:model=>model}, :layout => false if model.correct_predictions
         - when "regression"
           = haml :regression_validation, :locals=>{:model=>model}, :layout => false     
-    - else
-      = image_tag("/snake_transparent.gif") if model.validation_status == "Running"
-      %a{:href => model.validation_task_uri} #{model.validation_status}
-          
+-# else
+  = image_tag("/snake_transparent.gif") if model.validation_status == "Running"
+  %a{:href => model.validation_task_uri} #{model.validation_status}
+