working version with proxy

12 files changed, 359 insertions, 511 deletions

diff --git a/Gemfile b/Gemfile
index 1dff064..6e84031 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,8 +1,10 @@
 source :gemcutter
 gemspec
 gem "haml"
-gem "dalli"
-gem "rack-cache"
+#gem "dalli"
+#gem "rack-cache"
+gem "rest-client"
 gem "rest-client-components"
-gem "opentox-server", :path => "../opentox-server"
-gem "opentox-client", :path => "../opentox-client"
+gem "memcache-client"
+#gem "opentox-server", :path => "../opentox-server"
+#gem "opentox-client", :path => "../opentox-client"
diff --git a/application.rb b/application.rb
index 68bb8d2..9162fcf 100644
--- a/application.rb
+++ b/application.rb
@@ -1,25 +1,21 @@
+require "./pug.rb"
 require "./pubchem.rb"
-require 'rack/session/dalli'
-require "rack/cache"
 module OpenTox
-  class Application < Service
+  class Application < Sinatra::Base
+
     set :static, true
     set :root, File.dirname(__FILE__)
-    also_reload './pubchem.rb'
-
-    @@pug_uri = "http://pubchem.ncbi.nlm.nih.gov/rest/pug/"
 
-    before do
-      #cache_control :public, :max_age => 3600
+    configure :development do
+      register Sinatra::Reloader
+      also_reload './pubchem.rb'
     end
 
     before '/cid/:cid/*' do
-      #cache_control :public, :max_age => 3600
-      session[:compound] = PubChemCompound.new params[:cid] unless session[:compound] and session[:compound].cid == params[:cid]
+      @compound = PubChemCompound.new params[:cid] 
     end
 
     get '/?' do
-      #cache_control :public, :no_cache
       haml :index
     end
 
@@ -28,58 +24,71 @@ module OpenTox
     end
 
     get '/search/?' do
-      #cache_control :public, :no_cache
-      begin
-        cids = RestClient.get(File.join(@@pug_uri,"compound","name",CGI.escape(params[:name]),"cids","TXT")).split("\n")
-        if cids.size == 1
-          session[:compound] = PubChemCompound.new cids.first
-          haml :compound
-        elsif cids.size > 1
-          @compounds = cids.collect{|cid| PubChemCompound.new cid }
-          haml :select
-        end
-      rescue
+      @compounds = PubChemCompound.from_name params[:name]
+      if @compounds.nil?
         haml :not_found
+      elsif @compounds.is_a? Array
+        haml :select
+      else
+        @compound = @compounds
+        haml :compound
       end
     end
 
     get '/cid/:cid/targets/?' do
-      @assays = session[:compound].targets
-      haml :targets, :layout => false
+      @assays = @compound.targets
+      if @assays.empty?
+        "<br><em>No PubChem data</em></br>"
+      else
+        haml :targets, :layout => false
+      end
     end
 
     get '/cid/:cid/nontargets/?' do
-      @assays = session[:compound].non_targets
-      haml :targets, :layout => false
+      @assays = @compound.non_targets
+      if @assays.empty?
+        "<br><em>No PubChem data</em></br>"
+      else
+        haml :targets, :layout => false
+      end
     end
 
     get '/cid/:cid/other_active_assays/?' do
-      @assays = session[:compound].active_assays - session[:compound].targets
-      haml :assays, :layout => false
+      @assays = @compound.active_assays - @compound.targets
+      if @assays.empty?
+        "<br><em>No PubChem data</em></br>"
+      else
+        haml :assays, :layout => false
+      end
     end
 
     get '/cid/:cid/other_inactive_assays/?' do
-      @assays = session[:compound].inactive_assays - session[:compound].non_targets
-      haml :assays, :layout => false
+      @assays = @compound.inactive_assays - @compound.non_targets
+      if @assays.empty?
+        "<br><em>No PubChem data</em></br>"
+      else
+        haml :assays, :layout => false
+      end
     end
 
     get '/cid/:cid/predicted_targets/?' do
-      @assays = session[:compound].predicted_targets
+      @assays = @compound.predicted_targets
+      puts @assays.inspect
       haml :predicted_targets, :layout => false
     end
 
     get '/cid/:cid/predicted_nontargets/?' do
-      @assays = session[:compound].predicted_non_targets
+      @assays = @compound.predicted_non_targets
       haml :predicted_targets, :layout => false
     end
 
     get '/cid/:cid/other_predicted_active_assays/?' do
-      @assays = session[:compound].predicted_active_assays - session[:compound].predicted_targets
+      @assays = @compound.predicted_active_assays - @compound.predicted_targets
       haml :predicted_assays, :layout => false
     end
 
     get '/cid/:cid/other_predicted_inactive_assays/?' do
-      @assays = session[:compound].predicted_inactive_assays - session[:compound].predicted_non_targets
+      @assays = @compound.predicted_inactive_assays - @compound.predicted_non_targets
       haml :predicted_assays, :layout => false
     end
 
@@ -88,23 +97,8 @@ module OpenTox
     end
 
     get '/cid/:cid/cosine/:cid2/?' do
-      session[:compound].cosine(PubChemCompound.new(params[:cid2])).round(3).to_s
-    end
-
-=begin
-    get '/aid/:aid/?' do
-      puts File.join(@@pug_uri, "assay", "aid", params[:aid].to_s, "description", "JSON")
-      json = RestClient.get File.join(@@pug_uri, "assay", "aid", params[:aid].to_s, "description", "JSON")
-      @description = JSON.parse(json)["PC_AssayContainer"][0]["assay"]["descr"]
-      haml :assay_description, :layout => false
-    end
-
-    get '/pubchem_proxy/*' do |path|
-      puts path.inspect
-      puts "http://pubchem.ncbi.nlm.nih.gov/rest/pug/#{path}"
-      RestClientWrapper.get "http://pubchem.ncbi.nlm.nih.gov/rest/pug/#{path}"
+      @compound.cosine(PubChemCompound.new(params[:cid2])).round(3).to_s
     end
-=end
 
     get '/fp/?' do
       @fp = []
diff --git a/config.ru b/config.ru
index 76b7b7a..d6cf63b 100644
--- a/config.ru
+++ b/config.ru
@@ -2,7 +2,7 @@ SERVICE = "aop"
 require 'bundler'
 Bundler.require
 require './application.rb'
-use Rack::Session::Dalli, :cache => Dalli::Client.new
+#use Rack::Session::Dalli, :cache => Dalli::Client.new
 #use Rack::Cache,
 #  :verbose => true,
 #  :metastore   => "memcached://127.0.0.1:11211/meta",
diff --git a/pubchem.rb b/pubchem.rb
index ea72553..d16f4b4 100644
--- a/pubchem.rb
+++ b/pubchem.rb
@@ -1,13 +1,6 @@
 require '../opentox-client/lib/opentox-client.rb'
 require 'json'
 require 'base64'
-require 'restclient/components'
-require 'rack/cache'
-#RestClient.enable Rack::Cache, :verbose => true#, :allow_reload => true, :allow_revalidate => true
-RestClient.enable Rack::Cache, 
-  :verbose => true,
-  :metastore => 'file:/tmp/cache/meta', 
-  :entitystore => 'file:/tmp/cache/body'
 
 def Math.gauss(x, sigma = 0.3) 
   d = 1.0 - x.to_f
@@ -16,100 +9,40 @@ end
 
 module OpenTox
 
-  # doc @ http://pubchem.ncbi.nlm.nih.gov/pug_rest/
   class PubChemCompound < Compound
-    attr_writer :cid
-    attr_accessor :similarity, :p, :assays
+ 
+    attr_accessor :cid
+    @@pug_proxy = "http://localhost:8081/"
     
-    def initialize cid=nil
-      #@pug_uri = "http://pubchem.ncbi.nlm.nih.gov/rest/pug/"
-      @pug_uri = "http://localhost:8081/"
-      @cid = cid
-      @assays = nil
-      @similarity_threshold = 90
-      @neighbors = nil
-      @predicted_assays = nil
-      #@predicted_targets = nil
-      #@priors = {}
-      #@priors = JSON.parse(File.read("priors.json"))
+    def initialize cid
+      @cid = cid.to_s
     end
 
     def fingerprint
-      unless @fingerprint
-        begin
-          # ftp://ftp.ncbi.nlm.nih.gov/pubchem/specifications/pubchem_fingerprints.txt
-          base64key = `curl http://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/cid/#{cid}/SDF|grep -A1 PUBCHEM_CACTVS_SUBSKEYS|sed '1d'`.chomp
-          @fingerprint = Base64.decode64(base64key)[4..-1].unpack("B*").first[0..-8].split(//).collect{|c| c == "1"}
-        rescue
-        end
-      end
-      @fingerprint
+      JSON.parse RestClient.get(File.join(@@pug_proxy,"cid",@cid,"fingerprint"))
     end
 
     def self.from_name name
-      pug_uri = "http://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name"
-      compounds = []
-      session[:name] = name
-      cid = RestClient.get(File.join(pug_uri,URI.escape(name),"cids","TXT"))
-      #puts response
-      #response.split("\n") do |cid|
-        puts cid
-        compound = OpenTox::PubChemCompound.new
-        compound.cid = cid.chomp
-        compounds << compound
-      #end
-      compounds
+      cids = JSON.parse(RestClient.get(File.join(@@pug_proxy,"name",CGI.escape(name))))
+      if cids.size == 1
+        PubChemCompound.new cids.first
+      elsif cids.empty?
+        nil
+      else
+        cids.collect{|cid| PubChemCompound.new cid}
+      end
     end
 
     def name
-      RestClient.get File.join(@pug_uri, "compound", "cid", cid.to_s, "property", "IUPACName","TXT").chomp
+      RestClient.get(File.join(@@pug_proxy,"cid",cid,"name")).chomp.sub(/^"/,'').sub(/"$/,'')
     end
 
     def neighbors
-      unless @neighbors
-        @neighbors = []
-        result = pubchem_search File.join(@pug_uri, "compound", "similarity", "cid", cid.to_s, "JSON")+"?Threshold=#{@similarity_threshold}&MaxRecords=100"
-        while result["Waiting"] do
-          sleep 2
-          listkey = result["Waiting"]["ListKey"]
-          result = pubchem_search File.join(@pug_uri, "compound", "listkey", listkey, "cids", "JSON")
-          #result = pubchem_search File.join(@pug_uri, "compound", "listkey", listkey, "assaysummary", "JSON")
-        end
-        puts "#{result["IdentifierList"]["CID"].size} Neighbor CIDs received"
-        result["IdentifierList"]["CID"].each do |cid|
-          unless cid.to_s == @cid.to_s
-            c = PubChemCompound.new cid.to_s
-            @neighbors << c if c.assays and !c.assays.empty?
-          end
-        end if result and result["IdentifierList"]
-=begin
-        if result and result["Table"]
-          columns = result["Table"]["Columns"]["Column"]
-          table = result["Table"]["Row"].collect{|cell| cell.values.flatten}
-          cid_idx = columns.index("CID")
-          cids = table.collect{|r| r[cid_idx]}.uniq
-          cids.each do |cid|
-            unless cid.to_s == @cid.to_s
-              tab = table.collect{|r| r if r[cid_idx] == cid}.compact
-              c = PubChemCompound.new
-              c.extract_result columns, tab
-              c.similarity = tanimoto c
-              @neighbors << c unless (c.targets + c.non_targets).empty?
-            end
-          end
-        end
-=end
-        #@neighbors.sort!{|a,b| b.similarity <=> a.similarity}
-      end
-      @neighbors
+      JSON.parse(RestClient.get(File.join(@@pug_proxy,"cid",@cid,"neighbors"))).collect{|n| PubChemCompound.new(n) }
     end
 
     def assays
-      unless @assays
-        result = pubchem_search File.join(@pug_uri, "compound", "cid", cid.to_s, "assaysummary", "JSON")
-        extract_result result["Table"]["Columns"]["Column"], result["Table"]["Row"].collect{|cell| cell.values.flatten} if result and result["Table"]
-      end
-      @assays
+      JSON.parse RestClient.get(File.join(@@pug_proxy,"cid",cid,"assays"))
     end
 
     def active_assays
@@ -129,54 +62,15 @@ module OpenTox
     end
 
     def predicted_assays
-      unless @predicted_assays
-        @predicted_assays = []
-        neighbors.collect{|n| n.assays.collect{|a| a["AID"]}}.flatten.compact.uniq.each do |aid|
-          predicted_assay = {"AID" => aid}
-          neighbors.each do |neighbor|
-            if similarity(neighbor) and similarity(neighbor) > 0.5 # avoid downweighting
-              search = neighbor.assays.select{|a| a["AID"] == aid}
-              search.each do |assay|
-                predicted_assay["Target GI"] ||= assay["Target GI"]
-                predicted_assay["Target Name"] ||= assay["Target Name"]
-                predicted_assay["Assay Name"] ||= assay["Assay Name"]
-                predicted_assay[:active_similarities] ||= []
-                predicted_assay[:inactive_similarities] ||= []
-
-                if assay["Activity Outcome"] == "active"
-                  predicted_assay[:p_active] ? predicted_assay[:p_active] = predicted_assay[:p_active]*similarity(neighbor) : predicted_assay[:p_active] = similarity(neighbor)
-                  predicted_assay[:p_inactive] ? predicted_assay[:p_inactive] = predicted_assay[:p_inactive]*(1-similarity(neighbor)) : predicted_assay[:p_inactive] = 1-similarity(neighbor)
-                  predicted_assay[:active_similarities] << similarity(neighbor)
-                elsif assay["Activity Outcome"] == "inactive"
-                  predicted_assay[:p_active] ? predicted_assay[:p_active] = predicted_assay[:p_active]*(1-similarity(neighbor)) : predicted_assay[:p_active] = 1-similarity(neighbor)
-                  predicted_assay[:p_inactive] ? predicted_assay[:p_inactive] = predicted_assay[:p_inactive]*similarity(neighbor) : predicted_assay[:p_inactive] = similarity(neighbor)
-                  predicted_assay[:inactive_similarities] << similarity(neighbor)
-                end
-              end
-            end
-          end
-          if predicted_assay[:p_active] and predicted_assay[:p_inactive] and predicted_assay[:p_active] != 0 and predicted_assay[:p_inactive] != 0
-            predicted_assay[:p_active] = predicted_assay[:p_active]/(predicted_assay[:p_active]+predicted_assay[:p_inactive])
-            predicted_assay[:p_inactive] = predicted_assay[:p_inactive]/(predicted_assay[:p_active]+predicted_assay[:p_inactive])
-            if predicted_assay[:p_active] > predicted_assay[:p_inactive]
-              predicted_assay[:prediction] = "active"
-            elsif predicted_assay[:p_active] < predicted_assay[:p_inactive]
-              predicted_assay[:prediction] = "inactive"
-            end
-            @predicted_assays << predicted_assay
-          end
-        end
-        #@predicted_targets.sort{|a,b| b[:p_active] <=> a[:p_active]}
-      end
-      @predicted_assays
+      JSON.parse RestClient.get(File.join(@@pug_proxy,"cid",cid,"predictions"))
     end
 
     def predicted_active_assays
-      predicted_assays.select{|a| a[:prediction] == "active"} if predicted_assays
+      predicted_assays.select{|a| a["p_active"] > a["p_inactive"]} if predicted_assays
     end
 
     def predicted_inactive_assays
-      predicted_assays.select{|a| a[:prediction] == "inactive"} if predicted_assays
+      predicted_assays.select{|a| a["p_active"] < a["p_inactive"]} if predicted_assays
     end
 
     def predicted_targets
@@ -187,241 +81,16 @@ module OpenTox
       predicted_inactive_assays.select{|a| a["Target GI"]} if predicted_assays
     end
 
-    def to_smiles
-      RestClient.get(File.join(@pug_uri, "compound", "cid", cid.to_s, "property", "CanonicalSMILES", "TXT")).strip
-    end
-
     def image_uri
-      File.join @pug_uri, "compound", "cid", @cid, "PNG"#?record_type=3d&image_size=small"
+      File.join @@pug_proxy, "cid", @cid, "image"
     end
 
     def similarity compound
       cosine compound
     end
 
-    def tanimoto compound
-      if fingerprint and compound.fingerprint
-        m11 = 0.0
-        m1 = 0.0
-        fingerprint.each_index do |i|
-          m11 += 1 if (@fingerprint[i] and compound.fingerprint[i])
-          m1 += 1 if (@fingerprint[i] or compound.fingerprint[i]) 
-        end
-        m11/m1
-      end
-    end
-
     def cosine compound
-      if fingerprint and compound.fingerprint
-        m11 = 0.0
-        m01 = 0.0
-        m10 = 0.0
-        m00 = 0.0
-        fingerprint.each_index do |i|
-          m11 += 1 if (@fingerprint[i] and compound.fingerprint[i])
-          m01 += 1 if (!@fingerprint[i] and compound.fingerprint[i]) 
-          m10 += 1 if (@fingerprint[i] and !compound.fingerprint[i]) 
-          m00 += 1 if (!@fingerprint[i] and !compound.fingerprint[i]) 
-        end
-        m11/((m01+m11)*(m10+m11))**0.5
-      end
-    end
-
-=begin
-      f1 = File.open(File.join(".","tmp",SecureRandom.uuid+".smi"),"w+")
-      f1.puts to_smiles
-      f1.close
-      f2 = File.open(File.join(".","tmp",SecureRandom.uuid+".smi"),"w+")
-      f2.puts compound.to_smiles
-      f2.close
-      sim = `babel #{f1.path} #{f2.path} -ofpt 2>/dev/null| grep Tanimoto|cut -d "=" -f2`.strip.to_f
-      File.delete(f1.path)
-      File.delete(f2.path)
-      sim
-    end
-=end
-
-    def pubchem_search url
-      attempts = 0
-      begin
-        attempts += 1
-        json = RestClient.get url, :timeout => 90000000
-        puts url
-        JSON.parse json
-      rescue
-        if $!.message =~ /Timeout/i and attempts < 4
-          sleep 2
-          retry
-        elsif $!.message =~ /Timeout/i and attempts >= 4
-          File.open("timeouts","a+"){|f| f.puts url}
-          puts url
-          puts $!.message
-          nil
-        elsif $!.message.match /404/
-          nil
-        else
-          puts url
-          puts $!.message
-          nil
-        end
-      end
-    end
-
-    def extract_result columns, table
-      @assays = []
-      table.each do |row|
-        @assays << {}
-        row.each_with_index do |cell,i|
-          if columns[i] == "CID" 
-            @cid = cell if @cid.nil?
-          else
-            cell.blank? ?  @assays.last[columns[i]] = nil : @assays.last[columns[i]] = cell
-          end
-        end
-      end
-    end
-
-    def priors aid
-      unless @priors[aid]
-        @priors[aid] = {"nr_active" => 0, "nr_inactive" => 0}
-        result = nil
-        result = pubchem_search File.join(@pug_uri, "assay", "aid", aid.to_s, "cids", "JSON?cids_type=active&list_return=listkey")
-        @priors[aid]["nr_active"] = result["IdentifierList"]["Size"].to_i if result
-        result = nil
-        result = pubchem_search File.join(@pug_uri, "assay", "aid", aid.to_s, "cids", "JSON?cids_type=inactive&list_return=listkey")
-        @priors[aid]["nr_inactive"] = result["IdentifierList"]["Size"].to_i if result
-        File.open("priors.json","w+"){|f| f.puts @priors.to_json}
-      end
-      @priors[aid]
-    end
-
-=begin
-    def assay_summary assay
-        if assay["Target GI"] and !@assays[assay["AID"]]
-          @assays[assay["AID"]] = {"nr_active" => 0, "nr_inactive" => 0}
-          pubchem_search File.join(@pug_uri, "assay", "aid", assay["AID"].to_s, "cids", "JSON?cids_type=active")
-          @assays[assay["AID"]]["nr_active"] = @result["InformationList"]["Information"].first["CID"].size if @result
-          pubchem_search File.join(@pug_uri, "assay", "aid", assay["AID"].to_s, "cids", "JSON?cids_type=inactive")
-          @assays[assay["AID"]]["nr_inactive"] = @result["InformationList"]["Information"].first["CID"].size if @result
-          print "getting (in)actives for aid "
-          puts assay["AID"]
-          print @assays[assay["AID"]]["nr_active"]
-          print " "
-          puts @assays[assay["AID"]]["nr_inactive"]
-          File.open("assays.json","w+"){|f| f.puts @assays.to_json}
-        end
-    end
-=end
-
-=begin
-
-    def properties
-      properties = [
-        "XLogP",
-        "ExactMass",
-        "MonoisotopicMass",
-        "TPSA",
-        "Complexity",
-        "Charge",
-        "HBondDonorCount",
-        "HBondAcceptorCount",
-        "RotatableBondCount",
-        "HeavyAtomCount",
-        "IsotopeAtomCount",
-        "AtomStereoCount",
-        "DefinedAtomStereoCount",
-        "UndefinedAtomStereoCount",
-        "BondStereoCount",
-        "DefinedBondStereoCount",
-        "UndefinedBondStereoCount",
-        "CovalentUnitCount",
-        "Volume3D",
-        "XStericQuadrupole3D",
-        "YStericQuadrupole3D",
-        "ZStericQuadrupole3D",
-        "FeatureCount3D",
-        "FeatureAcceptorCount3D",
-        "FeatureDonorCount3D",
-        "FeatureAnionCount3D",
-        "FeatureCationCount3D",
-        "FeatureRingCount3D",
-        "FeatureHydrophobeCount3D",
-        "ConformerModelRMSD3D",
-        "EffectiveRotorCount3D",
-        "ConformerCount3D",
-      ]
-      pubchem_search File.join(@pug_uri, "compound", "cid", @cid, "property", properties.join(","), "JSON")
-      @result["PropertyTable"]["Properties"].first
-    end
-
-    def from_smiles smiles
-      pubchem_search File.join(@pug_uri, "compound", "smiles", smiles, "assaysummary", "JSON")
-      extract_result @result["Table"]["Columns"]["Column"], @result["Table"]["Row"].collect{|cell| cell.values.flatten}
-    end
-    def property_similarity compound
-      svd = OpenTox::SVD.new(GSL::Matrix [[properties, compound.properties]])
-      OpenTox::Algorithm::Similarity.cosine svd.data_transformed_matrix.first, svd.data_transformed_matrix.last
-    end
-
-    def assay_similarity compound
-      tanimoto [[active_assays,inactive_assays],[compound.active_assays,compound.inactive_assays]]
-    end
-
-    def target_similarity compound
-      tanimoto [[targets,non_targets],[compound.targets,compound.non_targets]]
-    end
-
-    def tanimoto features
-      common = features.first.flatten & features.last.flatten
-      same_outcome = (features.first.first & features.last.first) + (features.first.last & features.last.last)
-      same_outcome.size.to_f/common.size
-    end
-
-    def euclid features
-    end
-
-    def to_name
-      RestClient.get(File.join(@pug_uri, "compound", "cid", @cid, "property", "IUPACName", "TXT")).strip
-    end
-=end
-
-  end
-
-=begin
-  class PubChemNeighbors < Dataset
-    include PubChem
-
-    attr_accessor :query, :neighbors
-
-    def initialize
-      @similarity_threshold = 95
-      @neighbors = []
-      @pug_uri = "http://pubchem.ncbi.nlm.nih.gov/rest/pug/"
-    end
-
-    def from_smiles smiles
-      #@query = PubChemCompound.new.from_smiles smiles
-      pubchem_search File.join(@pug_uri, "compound", "similarity", "smiles", smiles, "JSON")+"?Threshold=#{@similarity_threshold}&MaxRecords=250"
-      listkey = @result["Waiting"]["ListKey"]
-      while @result["Waiting"] do
-        sleep 1
-        pubchem_search File.join(@pug_uri, "compound", "listkey", listkey, "assaysummary", "JSON")
-      end
-      #File.open("search.yaml","w+"){|s| s.puts @result.to_yaml}
-      columns = @result["Table"]["Columns"]["Column"]
-      table = @result["Table"]["Row"].collect{|cell| cell.values.flatten}
-      cid_idx = columns.index("CID")
-      cids = table.collect{|r| r[cid_idx]}.uniq
-      cids.each do |cid|
-        tab = table.collect{|r| r if r[cid_idx] == cid}.compact
-        c = PubChemCompound.new
-        c.extract_result columns, tab
-        @neighbors << c unless (c.targets + c.active_assays).flatten.compact.empty?
-      end
-      @query = @neighbors.shift
-      File.open("search.yaml","w+"){|s| s.puts self.to_yaml}
-      #puts @neighbors.query.to_name
+      RestClient.get(File.join(@@pug_proxy,"cid",@cid,"cosine",compound.cid)).to_f
     end
   end
-=end
 end
diff --git a/pug.rb b/pug.rb
new file mode 100644
index 0000000..6a8b102
--- /dev/null
+++ b/pug.rb
@@ -0,0 +1,193 @@
+require "json"
+require 'base64'
+require 'sinatra/base'
+require "sinatra/reloader" 
+require "rest-client"
+require 'memcache'
+
+class Application < Sinatra::Base
+
+  # doc @ http://pubchem.ncbi.nlm.nih.gov/pug_rest/
+  @@pug_uri = "http://pubchem.ncbi.nlm.nih.gov/rest/pug/"
+  @@similarity_threshold = 90
+  @@max_neighbors = 100
+  
+  CACHE = MemCache.new 'localhost:11211'
+
+  helpers do
+
+    def local route
+      status, headers, body = call env.merge("PATH_INFO" => route)
+      begin
+        Float body[0]
+      rescue
+        JSON.parse body[0]
+      end
+    end
+
+    def pubchem_search url
+      attempts = 0
+      begin
+        attempts += 1
+        puts url
+        json = RestClient.get url, :timeout => 90000000
+        JSON.parse json
+      rescue
+        if $!.message =~ /Timeout/i and attempts < 4
+          sleep 2
+          retry
+        elsif $!.message =~ /Timeout/i and attempts >= 4
+          File.open("timeouts","a+"){|f| f.puts url}
+          puts url
+          puts $!.message
+          nil
+        elsif $!.message.match /404/
+          nil
+        else
+          puts url
+          puts $!.message
+          nil
+        end
+      end
+    end
+
+    def fingerprint cid
+      local("/cid/#{cid}/fingerprint")
+    end
+
+    def neighbors cid
+      local "/cid/#{cid}/neighbors"
+    end
+
+    def assays cid
+      local "/cid/#{cid}/assays"
+    end
+
+    def similarity cid1, cid2
+      local("/cid/#{cid1}/cosine/#{cid2}")
+    end
+
+    def cosine fp1, fp2
+      if fp1 and fp2
+        m11 = 0.0
+        m01 = 0.0
+        m10 = 0.0
+        m00 = 0.0
+        fp1.each_index do |i|
+          m11 += 1 if (fp1[i] and fp2[i])
+          m01 += 1 if (!fp1[i] and fp2[i]) 
+          m10 += 1 if (fp1[i] and !fp2[i]) 
+          m00 += 1 if (!fp1[i] and !fp2[i]) 
+        end
+        m11/((m01+m11)*(m10+m11))**0.5
+      end
+    end
+  end
+
+  before do
+    @result = CACHE.get request.path
+    halt 200, @result unless @result.nil? # should be 304, but this does not work with local()
+  end
+
+  after do
+    CACHE.add request.path, @result, 7200
+  end
+
+  get '/cid/:cid/name' do
+    @result = RestClient.get(File.join(@@pug_uri, "compound", "cid", params[:cid], "property", "IUPACName","TXT")).chomp
+  end
+
+  get '/cid/:cid/fingerprint' do
+    # ftp://ftp.ncbi.nlm.nih.gov/pubchem/specifications/pubchem_fingerprints.txt
+    # it seems that only SDF formats contain fingerprints 
+    sdf_lines = RestClient.get(File.join(@@pug_uri, "compound", "cid", params[:cid], "SDF")).split("\n")
+    index = sdf_lines.index(sdf_lines.grep(/PUBCHEM_CACTVS_SUBSKEYS/).first)
+    @result = Base64.decode64(sdf_lines[index+1])[4..-1].unpack("B*").first[0..-8].split(//).collect{|c| c == "1"}.to_json
+  end
+
+  get '/cid/:cid/assays' do
+    assays = []
+    result = pubchem_search File.join(@@pug_uri, "compound", "cid", params[:cid], "assaysummary", "JSON")
+    if result and result["Table"]
+      columns = result["Table"]["Columns"]["Column"]
+      result["Table"]["Row"].collect{|cell| cell.values.flatten}.each do |row|
+        assay = {}
+        row.each_with_index do |cell,i|
+          assay[columns[i]] = cell unless cell.empty? or columns[i] == "CID" 
+        end
+        assays << assay unless assay.empty?
+      end
+    end
+    @result = assays.to_json
+  end
+
+  get '/cid/:cid/neighbors' do
+    result = pubchem_search File.join(@@pug_uri, "compound", "similarity", "cid", params[:cid], "JSON")+"?Threshold=#{@@similarity_threshold}&MaxRecords=#{@@max_neighbors}"
+    while result["Waiting"] do
+      sleep 2
+      listkey = result["Waiting"]["ListKey"]
+      result = pubchem_search File.join(@@pug_uri, "compound", "listkey", listkey, "cids", "JSON")
+    end
+    result["IdentifierList"]["CID"].delete params[:cid].to_i
+    @result = result["IdentifierList"]["CID"].to_json
+  end
+
+  get '/name/:name' do
+    @result = RestClient.get(File.join(@@pug_uri,"compound","name",CGI.escape(params[:name]),"cids","TXT")).split("\n").to_json
+  end
+
+  get '/cid/:cid/image' do
+    @result = RestClient.get File.join(@@pug_uri, "compound", "cid", params[:cid], "PNG")
+  end
+
+  get '/cid/:cid1/cosine/:cid2' do
+    fp1 = fingerprint params[:cid1]
+    fp2 = fingerprint params[:cid2]
+    @result = cosine(fp1, fp2).to_json
+  end
+
+  get '/cid/:cid/predictions' do
+    assays = {}
+    assay_details = {}
+    neighbors(params[:cid]).each do |cid|
+      neighbor_assays = assays cid
+      unless neighbor_assays.empty?
+        neighbor_assays.each do |assay|
+          if assay["Activity Outcome"] == "active" or assay["Activity Outcome"] == "inactive"
+            assays[assay["AID"]] ||= []
+            assays[assay["AID"]] << [cid,similarity(params[:cid],cid),assay["Activity Outcome"]]
+            assay_details[assay["AID"]] ||= {}
+            ["Target GI", "Target Name", "Assay Name"].each do |d|
+              assay_details[assay["AID"]][d] = assay[d] #if assay[d]
+            end
+          end
+        end
+      end
+    end
+    predictions = []
+    assays.each do |aid,neighbors|
+      prediction = {"AID" => aid}
+      neighbors.each do |neighbor|
+        cid = neighbor[0]
+        sim = neighbor[1]
+        activity = neighbor[2]
+        if activity == "active"
+          prediction[:p_active] ? prediction[:p_active] = prediction[:p_active]*sim : prediction[:p_active] = sim
+          prediction[:p_inactive] ? prediction[:p_inactive] = prediction[:p_inactive]*(1-sim) : prediction[:p_inactive] = 1-sim
+        elsif activity == "inactive"
+          prediction[:p_active] ? prediction[:p_active] = prediction[:p_active]*(1-sim) : prediction[:p_active] = 1-sim
+          prediction[:p_inactive] ? prediction[:p_inactive] = prediction[:p_inactive]*sim : prediction[:p_inactive] = sim
+        end
+        ["Target GI", "Target Name", "Assay Name"].each do |d|
+          prediction[d] = assay_details[aid][d] if assay_details[aid][d]
+        end
+      end
+      predictions << prediction
+    end
+    @result = predictions.to_json
+  end
+
+#  get '/*' do |path|
+#    RestClient.get File.join(@@pug_uri,path), params
+#  end
+end
diff --git a/views/compound.haml b/views/compound.haml
index de4cf77..9509ce0 100644
--- a/views/compound.haml
+++ b/views/compound.haml
@@ -1,65 +1,41 @@
-%script{:type => "text/javascript", :src  => "sorttable.js"}
-%table{:class => "sortable"}
-  %tr
-    %th Structure
-    %th Targets (experimental data)
-    %th Other active assays (experimental data)
+:javascript
+  $(document).ready(function() {
+    /*/ prefetch predictions in order to fill cache in background
+    $.ajax({ 
+      url: "/cid/#{@compound.cid}/predicted_targets",
+      cache: true,
+      dataType: "html"
+    });
+    */
+    hide("Measured gene/protein targets",".targets", "/cid/#{@compound.cid}/targets");
+    hide("Other active assays",".active_assays", "/cid/#{@compound.cid}/other_active_assays");
+    hide("Measured gene/protein non-targets",".nontargets", "/cid/#{@compound.cid}/nontargets");
+    hide("Other inactive assays",".inactive_assays", "/cid/#{@compound.cid}/other_inactive_assays");
+    hide("Read across gene/protein targets",".predicted_targets", "/cid/#{@compound.cid}/predicted_targets");
+    hide("Other active read across assays",".predicted_active_assays", "/cid/#{@compound.cid}/other_predicted_active_assays");
+    hide("Read across gene/protein non-targets",".predicted_nontargets", "/cid/#{@compound.cid}/predicted_nontargets");
+    hide("Other inactive read across assays",".predicted_inactive_assays", "/cid/#{@compound.cid}/other_predicted_inactive_assays");
+    hide("Similar compounds",".neighbors", "/cid/#{@compound.cid}/neighbors");
+  });
+
+%table
+  %colgroup
+    %col{:width => "25%"}
+    %col{:width => "37%"}
+    %col{:width => "37%"}
   %tr
     %td{:valign => "top"}
-      %br= session[:compound].name
-      %img{:src => session[:compound].image_uri}
+      %br= @compound.name
+      %img{:src => @compound.image_uri}
     %td{:valign => "top"}
       .targets
-      :javascript
-        display(".targets", "/cid/#{session[:compound].cid}/targets");
-    %td{:valign => "top"}
-      .active_assays
-      :javascript
-        display(".active_assays", "/cid/#{session[:compound].cid}/other_active_assays");
-  %tr
-    %th 
-    %th Targets (predicted)
-    %th Other active assays (predicted)
-  %tr
-    %td
-    %td{:valign => "top"}
       .predicted_targets
-      :javascript
-        display(".predicted_targets", "/cid/#{session[:compound].cid}/predicted_targets");
-    %td{:valign => "top"}
-      .predicted_active_assays
-      :javascript
-        display(".predicted_active_assays", "/cid/#{session[:compound].cid}/other_predicted_active_assays");
-  %tr
-    %th 
-    %th Non-targets (experimental data)
-    %th Other inactive assays (experimental data)
-  %tr
-    %td
-    %td{:valign => "top"}
       .nontargets
-      :javascript
-        display(".nontargets", "/cid/#{session[:compound].cid}/nontargets");
-    %td{:valign => "top"}
-      .inactive_assays
-      :javascript
-        display(".inactive_assays", "/cid/#{session[:compound].cid}/other_inactive_assays");
-  %tr
-    %th 
-    %th Non-targets (predicted)
-    %th Other inactive assays (predicted)
-  %tr
-    %td
-    %td{:valign => "top"}
       .predicted_nontargets
-      :javascript
-        display(".predicted_nontargets", "/cid/#{session[:compound].cid}/predicted_nontargets");
     %td{:valign => "top"}
+      .active_assays
+      .predicted_active_assays
+      .inactive_assays
       .predicted_inactive_assays
-      :javascript
-        display(".predicted_inactive_assays", "/cid/#{session[:compound].cid}/other_predicted_inactive_assays");
 
-%h2 Read across compounds
 .neighbors
-:javascript
-  display(".neighbors", "/cid/#{session[:compound].cid}/neighbors");
diff --git a/views/layout.haml b/views/layout.haml
index 40c6747..1abb7a8 100644
--- a/views/layout.haml
+++ b/views/layout.haml
@@ -3,6 +3,26 @@
   %head
     %script{:type => "text/javascript", :src  => "jquery-1.8.2.js"}
     :javascript
+      function show(title,element,uri) {
+        $(element).html("<h4>"+title+"</h4>"+"<img src=\"/spinning-wait-icons/wait30trans.gif\" alt=\"Searching PubChem\">");
+        $.ajax({
+          cache: true,
+          url: uri,
+          success: function(data){
+            data = "<h4>"+title+"</h4>"+"<button onclick='hide(\"" + title + "\",\"" + element + "\",\"" + uri + "\");'>Hide</button>" + data;
+            $(element).html(data);
+          },
+          error: function(data,textStatus,message){
+            $(element).html(message);
+          }
+        });
+      }
+
+      function hide(title,element,uri) {
+        data = "<h4>"+title+"</h4>"+"<button onclick='show(\"" + title + "\",\""  + element + "\",\"" + uri + "\");'>Show</button>";
+        $(element).html(data);
+      }
+
       function display(element,uri) {
         $(element).html("<img src=\"/spinning-wait-icons/wait30trans.gif\" alt=\"Searching PubChem\">");
         $.ajax({
@@ -18,7 +38,7 @@
       }
 
   %body
-    %h1 adverse outcome pathways
+    %h1 AOP read across
     %form{:name => "form", :action => '/search', :method => "GET"}
       %fieldset
         %label{:for => 'name'} Compound name:
diff --git a/views/neighbors.haml b/views/neighbors.haml
index 7b9c79d..09dc6cf 100644
--- a/views/neighbors.haml
+++ b/views/neighbors.haml
@@ -1,37 +1,33 @@
 %table
-  - session[:compound].neighbors[0..10].each do |compound|
-    %tr
-      %th Structure
-      %th Similarity
-      %th Targets (experimental data)
-      %th Other active assays (experimental data)
-
-    %tr
-      %td{:valign => "top"}
-        %br= compound.name
-        %img{:src => compound.image_uri}
-      %td{:id => "sim#{compound.cid}", :valign => "top"}
-        :javascript
-          display("#sim#{compound.cid}", "/cid/#{session[:compound].cid}/cosine/#{compound.cid}");
-      %td{:id => "targets#{compound.cid}", :valign => "top"}
-        :javascript
-          display("#targets#{compound.cid}", "/cid/#{compound.cid}/targets");
-      %td{:id => "assays#{compound.cid}", :valign => "top"}
-        :javascript
-          display("#assays#{compound.cid}", "/cid/#{compound.cid}/other_active_assays");
-    
-    %tr
-      %th 
-      %th 
-      %th Non-targets (experimental data)
-      %th Other inactive assays (experimental data)
-
-    %tr
-      %td
-      %td
-      %td{:id => "targets#{compound.cid}", :valign => "top"}
-        :javascript
-          display("#targets#{compound.cid}", "/cid/#{compound.cid}/nontargets");
-      %td{:id => "assays#{compound.cid}", :valign => "top"}
-        :javascript
-          display("#assays#{compound.cid}", "/cid/#{compound.cid}/other_inactive_assays");
+  %colgroup
+    %col{:width => "25%"}
+    %col{:width => "37%"}
+    %col{:width => "37%"}
+  - idx = 0
+  - while idx < 10
+    - @compound.neighbors.each do |n|
+      - unless n.assays.empty?
+        %tr
+          %td{:valign => "top"}
+            %br
+              = n.name
+              (
+              = @compound.cosine(n).round(3)
+              )
+            %img{:src => n.image_uri}
+          %td{:valign => "top"}
+            %p{:id => "targets#{n.cid}"}
+              :javascript
+                hide("Measured gene/protein targets","#targets#{n.cid}", "/cid/#{n.cid}/targets");
+            %p{:id => "nontargets#{n.cid}"}
+              :javascript
+                hide("Measured gene/protein non-targets","#nontargets#{n.cid}", "/cid/#{n.cid}/nontargets");
+          %td{:valign => "top"}
+            %p{:id => "assays#{n.cid}"}
+              :javascript
+                hide("Other active assays","#assays#{n.cid}", "/cid/#{n.cid}/other_active_assays");
+            %p{:id => "inactive_assays#{n.cid}"}
+              :javascript
+                hide("Other inactive assays","#inactive_assays#{n.cid}", "/cid/#{n.cid}/other_inactive_assays");
+        
+        - idx += 1
diff --git a/views/nontargets.haml b/views/nontargets.haml
deleted file mode 100644
index 154b5da..0000000
--- a/views/nontargets.haml
+++ /dev/null
@@ -1,3 +0,0 @@
-%pre
-  =# session[:compound].non_targets.to_yaml
-
diff --git a/views/not_found.haml b/views/not_found.haml
index 5903559..6b3485f 100644
--- a/views/not_found.haml
+++ b/views/not_found.haml
@@ -1,2 +1,3 @@
-No compound found that matches name 
-= "\"#{params[:name]}\"."
+Could not find a compound with name 
+= "\"#{params[:name]}\""
+in PubChem.
diff --git a/views/predicted_assays.haml b/views/predicted_assays.haml
index 991c267..299abf4 100644
--- a/views/predicted_assays.haml
+++ b/views/predicted_assays.haml
@@ -1,5 +1,5 @@
 %dl
-  - @assays.sort{|a,b| [b[:p_active],b[:p_inactive]].max <=> [a[:p_active],a[:p_inactive]].max}.each do |assay|
+  - @assays.sort{|a,b| [b["p_active"],b["p_inactive"]].max <=> [a["p_active"],a["p_inactive"]].max}.each do |assay|
     %dt
       %a{:href => "http://pubchem.ncbi.nlm.nih.gov/assay/assay.cgi?aid=#{assay["AID"]}"} #{assay['Assay Name']}
     %dd
@@ -7,7 +7,7 @@
       %a{:href => "http://pubchem.ncbi.nlm.nih.gov/assay/assay.cgi?aid=#{assay["AID"]}"} #{assay['AID']}
     %dd
       p_active:
-      = assay[:p_active].to_f.round(3)
+      = assay["p_active"].to_f.round(3)
     %dd
       p_inactive:
-      = assay[:p_inactive].to_f.round(3)
+      = assay["p_inactive"].to_f.round(3)
diff --git a/views/predicted_targets.haml b/views/predicted_targets.haml
index 523a217..97c68c4 100644
--- a/views/predicted_targets.haml
+++ b/views/predicted_targets.haml
@@ -1,15 +1,15 @@
 %dl
-  - @assays.sort{|a,b| [b[:p_active],b[:p_inactive]].max <=> [a[:p_active],a[:p_inactive]].max}.each do |assay|
+  - @assays.sort{|a,b| [b["p_active"],b["p_inactive"]].max <=> [a["p_active"],a["p_inactive"]].max}.each do |assay|
     %dt= assay["Target Name"]
     %dd
       Target GeneID:
       = assay["Target GI"]
     %dd
       Assay ID:
-      %a{:href => "http://pubchem.ncbi.nlm.nih.gov/assay/assay.cgi?aid=#{assay["AID"]}"} #{assay['AID']}
+      %a{:href => "http://pubchem.ncbi.nlm.nih.gov/assay/assay.cgi?aid=#{assay["AID"]}"} #{assay["AID"]}
     %dd
       p_active:
-      = assay[:p_active].to_f.round(3)
+      = assay["p_active"].to_f.round(3)
     %dd
       p_inactive:
-      = assay[:p_inactive].to_f.round(3)
+      = assay["p_inactive"].to_f.round(3)