code cleanup and refactoring.

21 files changed, 653 insertions, 1025 deletions

diff --git a/lib/4store.rb b/lib/4store.rb
deleted file mode 100644
index 338573d..0000000
--- a/lib/4store.rb
+++ /dev/null
@@ -1,126 +0,0 @@
-module OpenTox
-  module Backend
-    class FourStore
-
-      @@accept_formats = [ "application/rdf+xml", "text/turtle", "text/plain", "text/uri-list", "text/html", 'application/sparql-results+xml' ]
-      @@content_type_formats = [ "application/rdf+xml", "text/turtle", "text/plain", "application/x-turtle" ]
-
-      def self.list mime_type
-        bad_request_error "'#{mime_type}' is not a supported mime type. Please specify one of #{@@accept_formats.join(", ")} in the Accept Header." unless @@accept_formats.include? mime_type
-        if mime_type =~ /(uri-list|html)/
-          sparql = "SELECT DISTINCT ?g WHERE {GRAPH ?g {?s <#{RDF.type}> <#{klass}>; <#{RDF::DC.modified}> ?o.} } ORDER BY ?o"
-        else 
-          sparql = "CONSTRUCT {?s ?p ?o.} WHERE {?s <#{RDF.type}> <#{klass}>; ?p ?o. }"
-        end
-        query sparql, mime_type
-      end
-
-      def self.get uri, mime_type
-        bad_request_error "'#{mime_type}' is not a supported mime type. Please specify one of #{@@accept_formats.join(", ")} in the Accept Header." unless @@accept_formats.include? mime_type
-        sparql = "CONSTRUCT {?s ?p ?o.} FROM <#{uri}> WHERE { ?s ?p ?o. }"
-        rdf = query sparql, mime_type
-        resource_not_found_error "#{uri} not found." if rdf.empty?
-        rdf
-      end
-
-      def self.post uri, rdf, mime_type
-        bad_request_error "'#{mime_type}' is not a supported content type. Please use one of #{@@content_type_formats.join(", ")}." unless @@content_type_formats.include? mime_type or mime_type == "multipart/form-data"
-        bad_request_error "Reqest body empty." unless rdf 
-        mime_type = "application/x-turtle" if mime_type == "text/plain" # ntriples is turtle in 4store
-        begin
-          RestClient.post File.join(four_store_uri,"data")+"/", :data => rdf, :graph => uri, "mime-type" => mime_type 
-          update "INSERT DATA { GRAPH <#{uri}> { <#{uri}> <#{RDF::DC.modified}> \"#{DateTime.now}\" } }"
-        rescue
-          bad_request_error $!.message, File.join(four_store_uri,"data")+"/"
-        end
-      end
-
-      def self.put uri, rdf, mime_type
-        bad_request_error "'#{mime_type}' is not a supported content type. Please use one of #{@@content_type_formats.join(", ")}." unless @@content_type_formats.include? mime_type
-        bad_request_error "Reqest body empty." unless rdf 
-        mime_type = "application/x-turtle" if mime_type == "text/plain"
-        begin
-          RestClientWrapper.put File.join(four_store_uri,"data",uri), rdf, :content_type => mime_type 
-          update "INSERT DATA { GRAPH <#{uri}> { <#{uri}> <#{RDF::DC.modified}> \"#{DateTime.now}\" } }"
-        rescue
-          bad_request_error $!.message, File.join(four_store_uri,"data",uri)
-        end
-      end
-
-      def self.delete uri
-        RestClientWrapper.delete data_uri(uri)
-      end
-
-      def self.update sparql
-        RestClient.post(update_uri, :update => sparql )
-      end
-
-      def self.query sparql, mime_type
-        if sparql =~ /SELECT/i
-          # return list unless mime_type
-          case mime_type
-          when 'application/sparql-results+xml' 
-            RestClient.get(sparql_uri, :params => { :query => sparql }, :accept => mime_type).body
-          when 'application/json'
-            RestClient.get(sparql_uri, :params => { :query => sparql }, :accept => mime_type).body
-          when /(uri-list|html)/
-            uri_list = RestClient.get(sparql_uri, :params => { :query => sparql }, :accept => "text/plain").body.gsub(/"|<|>/,'').split("\n").drop(1).join("\n")
-            uri_list = OpenTox.text_to_html(uri_list) if mime_type=~/html/
-            return uri_list
-          else
-            bad_request_error "#{mime_type} is not a supported mime type for SELECT statements."
-          end
-        elsif sparql =~ /CONSTRUCT/i
-          case mime_type
-          when "text/plain", "application/rdf+xml" 
-            RestClient.get(sparql_uri, :params => { :query => sparql }, :accept => mime_type).body
-          when /html|turtle/
-            # TODO: fix and improve
-            nt = RestClient.get(sparql_uri, :params => { :query => sparql }, :accept => "text/plain").body # 4store returns ntriples for turtle
-
-            rdf = RDF::Graph.new
-            RDF::Reader.for(:ntriples).new(nt) do |reader|
-              reader.each_statement { |statement| rdf << statement }
-            end
-            prefixes = {:rdf => "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"}
-            ['OT', 'DC', 'XSD', 'OLO'].each{|p| prefixes[p.downcase.to_sym] = eval("RDF::#{p}.to_s") }
-            # TODO: fails for large datasets?? multi_cell_call
-            turtle = RDF::N3::Writer.for(:turtle).buffer(:prefixes => prefixes)  do |writer|
-              rdf.each{|statement| writer << statement}
-            end
-            regex = Regexp.new '(https?:\/\/[\S]+)([>"])'
-            turtle =  "<html><body>" + turtle.gsub( regex, '<a href="\1">\1</a>\2' ).gsub(/\n/,'<br/>') + "</body></html>" if mime_type =~ /html/ and !turtle.empty?
-            turtle
-          end
-        else
-          # TODO: check if this prevents SPARQL injections
-          bad_request_error "Only SELECT and CONSTRUCT are accepted SPARQL statements."
-        end
-      rescue
-        bad_request_error $!.message, sparql_uri
-      end
-
-      def self.klass
-        RDF::OT[SERVICE.capitalize]
-      end
-
-      def self.four_store_uri
-        # credentials are removed from uri in error.rb
-        $four_store[:uri].sub(%r{//},"//#{$four_store[:user]}:#{$four_store[:password]}@")
-      end
-
-      def self.sparql_uri 
-        File.join(four_store_uri, "sparql") + '/'
-      end
-
-      def self.update_uri 
-        File.join(four_store_uri, "update") + '/'
-      end
-
-      def self.data_uri uri
-        File.join(four_store_uri, "data","?graph=#{uri}")
-      end
-
-    end
-  end
-end
diff --git a/lib/algorithm.rb b/lib/algorithm.rb
index 0c633f2..455e9ad 100644
--- a/lib/algorithm.rb
+++ b/lib/algorithm.rb
@@ -7,9 +7,9 @@ module OpenTox
     # @param [optional,Hash] params Algorithm parameters
     # @param [optional,OpenTox::Task] waiting_task (can be a OpenTox::Subtask as well), progress is updated accordingly
     # @return [String] URI of new resource (dataset, model, ...)
-    def run params=nil
-      post params
+    def run params=nil, wait=true
+      uri = RestClientWrapper.post @uri, params, { :content_type => "text/uri-list", :subjectid => @subjectid}
+      wait_for_task uri if wait
     end
-
   end
 end
diff --git a/lib/compound.rb b/lib/compound.rb
index a0d6ec7..5d77dec 100644
--- a/lib/compound.rb
+++ b/lib/compound.rb
@@ -10,23 +10,21 @@ module OpenTox
     #   compound = OpenTox::Compound.from_smiles("c1ccccc1")
     # @param [String] smiles Smiles string
     # @return [OpenTox::Compound] Compound
-    def self.from_smiles service_uri, smiles, subjectid=nil
-      #@smiles = smiles
+    def self.from_smiles smiles, subjectid=nil
       Compound.new RestClientWrapper.post(service_uri, smiles, {:content_type => 'chemical/x-daylight-smiles', :subjectid => subjectid})
     end
 
     # Create a compound from inchi string
     # @param [String] smiles InChI string
     # @return [OpenTox::Compound] Compound
-    def self.from_inchi service_uri, inchi, subjectid=nil
-      #@inchi = inchi
+    def self.from_inchi inchi, subjectid=nil
       Compound.new RestClientWrapper.post(service_uri, inchi, {:content_type => 'chemical/x-inchi', :subjectid => subjectid})
     end
 
     # Create a compound from sdf string
     # @param [String] smiles SDF string
     # @return [OpenTox::Compound] Compound
-    def self.from_sdf service_uri, sdf, subjectid=nil
+    def self.from_sdf sdf, subjectid=nil
       Compound.new RestClientWrapper.post(service_uri, sdf, {:content_type => 'chemical/x-mdl-sdfile', :subjectid => subjectid})
     end
 
@@ -36,7 +34,7 @@ module OpenTox
     # @param [String] name name can be also an InChI/InChiKey, CAS number, etc
     # @return [OpenTox::Compound] Compound
     #
-    def self.from_name service_uri, name, subjectid=nil
+    def self.from_name name, subjectid=nil
       @inchi = RestClientWrapper.get File.join(CACTUS_URI,URI.escape(name),"stdinchi")
       Compound.new RestClientWrapper.post(service_uri, @inchi, {:content_type => 'chemical/x-inchi', :subjectid => subjectid})
     end
@@ -216,7 +214,7 @@ module OpenTox
           end
         end
       end
-      return smarts_hits
+      smarts_hits
     end
 
     # Provided for backward compatibility
diff --git a/lib/dataset.rb b/lib/dataset.rb
index 55d5fa8..8d1aed0 100644
--- a/lib/dataset.rb
+++ b/lib/dataset.rb
@@ -5,160 +5,344 @@ module OpenTox
   # Ruby wrapper for OpenTox Dataset Webservices (http://opentox.org/dev/apis/api-1.2/dataset).
   class Dataset 
 
-    attr_accessor :features, :compounds, :data_entries
+    attr_writer :features, :compounds, :data_entries
 
     def initialize uri=nil, subjectid=nil
       super uri, subjectid
       @features = []
       @compounds = []
       @data_entries = []
-      append RDF.type, RDF::OT.OrderedDataset
     end
 
-    def upload filename, wait=true
-      uri = RestClientWrapper.put(@uri, {:file => File.new(filename)}, {:subjectid => @subjectid})
-      OpenTox::Task.new(uri).wait if URI.task?(uri) and wait
-    end
+    # Get data (lazy loading from dataset service)
 
-    def to_csv
-      CSV.generate do |csv|
-        csv << ["SMILES"] + @features.collect{|f| f.title}
-        @compounds.each_with_index do |c,i|
-          csv << [c.smiles] + @data_entries[i]
+    def metadata force_update=false
+      if @metadata.empty? or force_update
+        uri = File.join(@uri,"metadata")
+        begin
+          parse_ntriples RestClientWrapper.get(uri,{},{:accept => "text/plain", :subjectid => @subjectid})
+        rescue # fall back to rdfxml
+          parse_rdfxml RestClientWrapper.get(uri,{},{:accept => "application/rdf+xml", :subjectid => @subjectid})
         end
+        @metadata = @rdf.to_hash[RDF::URI.new(@uri)].inject({}) { |h, (predicate, values)| h[predicate] = values.collect{|v| v.to_s}; h }
       end
+      @metadata
     end
 
-    def get(force_no_backend_query=false)
-      have_rdf = (force_no_backend_query and @rdf.size>0)
-      ordered = (have_rdf or OpenTox::Dataset.ordered?(@uri))
-      super() if (!have_rdf and !ordered)
-      @features = []
-      @compounds = []
-      @data_entries = []
-
-      # AM: read ordered dataset from RDF
-      if ordered
+    def features force_update=false
+      if @features.empty? or force_update
+        uri = File.join(@uri,"features")
+        uris = RestClientWrapper.get(uri,{},{:accept => "text/uri-list", :subjectid => @subjectid}).split("\n") # ordered datasets return ordered features
+        @features = uris.collect{|uri| Feature.new(uri,@subjectid)}
+      end
+      @features
+    end
 
-        # Read only some data as rdf
-        unless have_rdf
-          self.parse_rdfxml( RestClient.get([@uri,"allnde"].join("/"),{:accept => "application/rdf+xml"}), true )
+    def compounds force_update=false
+      if @compounds.empty? or force_update
+        uri = File.join(@uri,"compounds")
+        uris = RestClientWrapper.get(uri,{},{:accept => "text/uri-list", :subjectid => @subjectid}).split("\n") # ordered datasets return ordered compounds
+        @compounds = uris.collect{|uri| Compound.new(uri,@subjectid)}
+      end
+      @compounds
+    end
+    
+    def data_entries force_update=false
+      if @data_entries.empty? or force_update
+        sparql = "SELECT ?cidx ?fidx ?value FROM <#{uri}> WHERE {
+          ?data_entry <#{RDF::OLO.index}> ?cidx ;
+                      <#{RDF::OT.values}> ?v .
+          ?v          <#{RDF::OT.feature}> ?f;
+                      <#{RDF::OT.value}> ?value .
+          ?f          <#{RDF::OLO.index}> ?fidx.
+          } ORDER BY ?fidx ?cidx" 
+          RestClientWrapper.get(service_uri,{:query => sparql},{:accept => "text/uri-list", :subjectid => @subjectid}).split("\n").each do |row|
+            r,c,v = row.split("\t")
+            @data_entries[r.to_i] ||= []
+            @data_entries[r.to_i][c.to_i] = v
+          end
+        # TODO: fallbacks for external and unordered datasets
+        features.each_with_index do |feature,i|
+          if feature[RDF.type].include? RDF::OT.NumericFeature
+            @data_entries.each { |row| row[i] = row[i].to_f if row[i] }
+          end
         end
+      end
+      @data_entries
+    end
 
-        # Features
-        @features = self.find_features_rdf
-        numeric_features = @features.collect{|f| 
-          f.get
-          f[RDF.type].include?(RDF::OT.NumericFeature) or f[RDF.type].include?(RDF::OT.Substructure)
-        }
-
-        # Compounds
-        if have_rdf
-          @compounds = self.find_compounds_rdf
-        else
-          @compounds = RestClient.get([@uri,"compounds"].join("/"),{:accept => "text/uri-list"}).split("\n").collect { |cmpd| OpenTox::Compound.new cmpd }
-        end
+    # Find data entry values for a given compound and feature 
+    # @param [OpenTox::Compound] Compound 
+    # @param [OpenTox::Feature] Feature 
+    # @return [Array] Data entry values
+    def values(compound, feature)
+      #puts compounds.inspect
+      #puts "=="
+      #puts compound.inspect
+      rows = (0 ... compounds.length).select { |r| compounds[r].uri == compound.uri }
+      #puts rows.inspect
+      col = features.collect{|f| f.uri}.index feature.uri
+      #puts col
+      #puts data_entries(true).inspect
+      rows.collect{|row| data_entries[row][col]}
+    end
 
-        # Data Entries
-        if have_rdf
-          table = self.find_data_entries_rdf
-        else
-          values = OpenTox::Dataset.find_data_entries_sparql(@uri)
-          table = values + Array.new(@compounds.size*@features.size-values.size, "")
-        end
-        
-        clim=(@compounds.size-1)
-        cidx = fidx = 0
-        num=numeric_features[fidx]
-        @data_entries = (Array.new(@compounds.size*@features.size)).each_slice(@features.size).to_a # init to nil
-        table.each { |val|
-          unless val.blank?
-            @data_entries[cidx][fidx] = (num ? val.to_f : val)
-          end
-          if (cidx < clim)
-            cidx+=1
-          else
-            cidx=0
-            fidx+=1
-            num=numeric_features[fidx]
-          end
-        }
+    # Convenience methods to search by compound/feature URIs
 
-      # AM: read unordered dataset from RDF
-      else
-        query = RDF::Query.new do
-          pattern [:uri, RDF.type, RDF::OT.Feature]
-        end
-        @features = query.execute(@rdf).collect{|s| OpenTox::Feature.new(s.uri.to_s)}
-        query = RDF::Query.new do
-          pattern [:data_entry, RDF::OT.compound, :compound]
+    # Search a dataset for a feature given its URI
+    # @param [String] Feature URI
+    # @return [OpenTox::Feature] Feature object, or nil if not present
+    def find_feature_uri(uri)
+      features.select{|f| f.uri == uri}.first
+    end
+
+    # Search a dataset for a compound given its URI
+    # @param [String] Compound URI
+    # @return [OpenTox::Compound] Compound object, or nil if not present
+    def find_compound_uri(uri)
+      compounds.select{|f| f.uri == uri}.first
+    end
+
+    def predictions
+      predictions = []
+      prediction_feature = nil
+      confidence_feature = nil
+      metadata[RDF::OT.predictedVariables].each do |uri|
+        feature = OpenTox::Feature.new uri, @subjectid
+        case feature.title
+        when /prediction$/
+          prediction_feature = feature
+        when /confidence$/
+          confidence_feature = feature
         end
-        @compounds = query.execute(@rdf).sort_by{|s| s.data_entry}.collect{|s| OpenTox::Compound.new s.compound.to_s}
-        numeric_features = @features.collect{|f| f.get; f[RDF.type].include? RDF::OT.NumericFeature}
-        @compounds.each do |compound|
-          values = []
-          @features.each_with_index do |feature,i|
-            query = RDF::Query.new do
-              pattern [:data_entry, RDF::OT.compound, RDF::URI.new(compound.uri)]
-              pattern [:data_entry, RDF::OT.values, :values]
-              pattern [:values, RDF::OT.feature, RDF::URI.new(feature.uri)]
-              pattern [:values, RDF::OT.value, :value]
-            end
-            value = query.execute(@rdf).first.value.to_s
-            value = value.to_f if numeric_features[i] and !value.nil?
-            values << value
-          end
-          @data_entries << values
+      end
+      if prediction_feature and confidence_feature
+        compounds.each do |compound|
+          value = values(compound,prediction_feature).first
+          confidence = values(compound,confidence_feature).first
+          predictions << {:compound => compound, :value => value, :confidence => confidence} if value and confidence
         end
       end
+      predictions
+    end
+
+    # Adding data (@features and @compounds are also writable)
+
+    def upload filename, wait=true
+      uri = RestClientWrapper.put(@uri, {:file => File.new(filename)}, {:subjectid => @subjectid})
+      wait_for_task uri if URI.task?(uri) and wait
+      metadata true 
+      @uri
     end
 
-    def get_metadata
-      uri = File.join(@uri,"metadata")
-      begin
-        parse_ntriples RestClientWrapper.get(uri,{},{:accept => "text/plain", :subjectid => @subjectid})
-      rescue # fall back to rdfxml
-        parse_rdfxml RestClientWrapper.get(uri,{},{:accept => "application/rdf+xml", :subjectid => @subjectid})
+    def add_data_entry compound, feature, value
+      @compounds << compound unless @compounds.collect{|c| c.uri}.include?(compound.uri)
+      row = @compounds.collect{|c| c.uri}.index(compound.uri)
+      @features << feature unless @features.collect{|f| f.uri}.include?(feature.uri)
+      col = @features.collect{|f| f.uri}.index(feature.uri)
+      @data_entries[row] ||= []
+      if @data_entries[row][col] # duplicated values
+        #row = @compounds.size
+        @compounds << compound
+        row = @compounds.collect{|c| c.uri}.rindex(compound.uri)
       end
-      metadata
+      @data_entries[row][col] = value
     end
 
-    def << data_entry
-      compound = data_entry.shift
-      bad_request_error "Dataset features are empty." unless features
-      bad_request_error "data_entry size '#{data_entry.size}' does not match features size '#{features.size}'." unless data_entry.size == features.size
-      bad_request_error "First data_entry is not a OpenTox::Compound" unless compound.class == OpenTox::Compound
+    # TODO: remove? might be dangerous if feature ordering is incorrect
+    def << row
+      compound = row.shift
+      bad_request_error "Dataset features are empty." unless @features
+      bad_request_error "Row size '#{row.size}' does not match features size '#{@features.size}'." unless row.size == @features.size
+      bad_request_error "First column is not a OpenTox::Compound" unless compound.class == OpenTox::Compound
       @compounds << compound
-      @data_entries << data_entry
+      @data_entries << row
+    end
+
+    # Serialisation
+
+    def to_csv
+      CSV.generate do |csv|
+        csv << ["SMILES"] + features.collect{|f| f.title}
+        compounds.each_with_index do |c,i|
+          csv << [c.smiles] + data_entries[i]
+        end
+      end
     end
 
     RDF_FORMATS.each do |format|
+      
+      # redefine rdf parse methods for all formats e.g. parse_rdfxml
+      send :define_method, "parse_#{format}".to_sym do |rdf|
+        # TODO: parse ordered dataset
+        # TODO: parse data entries
+        # TODO: parse metadata
+        @rdf = RDF::Graph.new
+        RDF::Reader.for(format).new(rdf) do |reader|
+          reader.each_statement{ |statement| @rdf << statement }
+        end
+        query = RDF::Query.new({ :uri => { RDF.type => RDF::OT.Compound } })
+        @compounds = query.execute(@rdf).collect { |solution| OpenTox::Compound.new solution.uri }
+        query = RDF::Query.new({ :uri => { RDF.type => RDF::OT.Feature } })
+        @features = query.execute(@rdf).collect { |solution| OpenTox::Feature.new solution.uri }
+        @compounds.each_with_index do |c,i|
+          @features.each_with_index do |f,j|
+          end
+        end
+      end
+
 
       # redefine rdf serialization methods 
       send :define_method, "to_#{format}".to_sym do
-        # TODO: check, might affect appending to unordered datasets
-        features.each_with_index do |feature,i|
+        @metadata[RDF.type] = RDF::OT.OrderedDataset 
+        create_rdf
+        @features.each_with_index do |feature,i|
           @rdf << [RDF::URI.new(feature.uri), RDF::URI.new(RDF.type), RDF::URI.new(RDF::OT.Feature)] 
           @rdf << [RDF::URI.new(feature.uri), RDF::URI.new(RDF::OLO.index), RDF::Literal.new(i)] 
         end
-        compounds.each_with_index do |compound,i|
+        @compounds.each_with_index do |compound,i|
           @rdf << [RDF::URI.new(compound.uri), RDF::URI.new(RDF.type), RDF::URI.new(RDF::OT.Compound)]
+          if defined? @neighbors and neighbors.include? compound
+            @rdf << [RDF::URI.new(compound.uri), RDF::URI.new(RDF.type), RDF::URI.new(RDF::OT.Neighbor)]
+          end
+
           @rdf << [RDF::URI.new(compound.uri), RDF::URI.new(RDF::OLO.index), RDF::Literal.new(i)]
           data_entry_node = RDF::Node.new
           @rdf << [RDF::URI.new(@uri), RDF::URI.new(RDF::OT.dataEntry), data_entry_node]
           @rdf << [data_entry_node, RDF::URI.new(RDF.type), RDF::URI.new(RDF::OT.DataEntry)]
           @rdf << [data_entry_node, RDF::URI.new(RDF::OLO.index), RDF::Literal.new(i)]
           @rdf << [data_entry_node, RDF::URI.new(RDF::OT.compound), RDF::URI.new(compound.uri)]
-          data_entries[i].each_with_index do |value,j|
+          @data_entries[i].each_with_index do |value,j|
             value_node = RDF::Node.new
             @rdf << [data_entry_node, RDF::URI.new(RDF::OT.values), value_node]
             @rdf << [value_node, RDF::URI.new(RDF::OT.feature), RDF::URI.new(@features[j].uri)]
             @rdf << [value_node, RDF::URI.new(RDF::OT.value), RDF::Literal.new(value)]
           end
         end
-        super()
+        RDF::Writer.for(format).buffer do |writer|
+          @rdf.each{|statement| writer << statement}
+        end
+      end
+
+    end
+
+=begin
+# TODO: fix bug that affects data_entry positions
+    def to_ntriples # redefined string version for better performance
+
+      ntriples = ""
+      @metadata[RDF.type] = [ RDF::OT.Dataset, RDF::OT.OrderedDataset ]
+      @metadata[RDF.type] ||= eval("RDF::OT."+self.class.to_s.split('::').last)
+      @metadata[RDF::DC.date] ||= DateTime.now
+      @metadata.each do |predicate,values|
+        [values].flatten.each { |value| ntriples << "<#{@uri}> <#{predicate}> '#{value}' .\n" }
+      end
+      @parameters.each do |parameter|
+        p_node = RDF::Node.new.to_s
+        ntriples <<  "<#{@uri}> <#{RDF::OT.parameters}> #{p_node} .\n"
+        ntriples <<  "#{p_node} <#{RDF.type}> <#{RDF::OT.Parameter}> .\n"
+        parameter.each { |k,v| ntriples <<  "#{p_node} <#{k}> '#{v}' .\n" }
+      end
+      @features.each_with_index do |feature,i|
+        ntriples <<  "<#{feature.uri}> <#{RDF.type}> <#{RDF::OT.Feature}> .\n" 
+        ntriples <<  "<#{feature.uri}> <#{RDF::OLO.index}> '#{i}' .\n" 
+      end
+      @compounds.each_with_index do |compound,i|
+        ntriples <<  "<#{compound.uri}> <#{RDF.type}> <#{RDF::OT.Compound}> .\n"
+        if defined? @neighbors and neighbors.include? compound
+          ntriples <<  "<#{compound.uri}> <#{RDF.type}> <#{RDF::OT.Neighbor}> .\n"
+        end
+
+        ntriples <<  "<#{compound.uri}> <#{RDF::OLO.index}> '#{i}' .\n"
+        data_entry_node = RDF::Node.new
+        ntriples <<  "<#{@uri}> <#{RDF::OT.dataEntry}> #{data_entry_node} .\n"
+        ntriples <<  "#{data_entry_node} <#{RDF.type}> <#{RDF::OT.DataEntry}> .\n"
+        ntriples <<  "#{data_entry_node} <#{RDF::OLO.index}> '#{i}' .\n"
+        ntriples <<  "#{data_entry_node} <#{RDF::OT.compound}> <#{compound.uri}> .\n"
+        @data_entries[i].each_with_index do |value,j|
+          value_node = RDF::Node.new
+          ntriples <<  "#{data_entry_node} <#{RDF::OT.values}> #{value_node} .\n"
+          ntriples <<  "#{value_node} <#{RDF::OT.feature}> <#{@features[j].uri}> .\n"
+          ntriples <<  "#{value_node} <#{RDF::OT.value}> '#{value}' .\n"
+        end
       end
+      ntriples
+
+    end
+=end
+
+    # Methods for for validation service
+    
+    def split( compound_indices, feats, metadata, subjectid=nil)
+      
+      bad_request_error "Dataset.split : Please give compounds as indices" if compound_indices.size==0 or !compound_indices[0].is_a?(Fixnum)
+      bad_request_error "Dataset.split : Please give features as feature objects (given: #{feats})" if feats!=nil and feats.size>0 and !feats[0].is_a?(OpenTox::Feature)
+      dataset = OpenTox::Dataset.new(nil, subjectid)
+      dataset.metadata = metadata
+      dataset.features = (feats ? feats : self.features)
+      compound_indices.each do |c_idx|
+        dataset << [ self.compounds[c_idx] ] + dataset.features.each_with_index.collect{|f,f_idx| self.data_entries[c_idx][f_idx]} 
+      end
+      dataset.put
+      dataset    
+    end
+    
 
+    # maps a compound-index from another dataset to a compound-index from this dataset
+    # mapping works as follows:
+    # (compound c is the compound identified by the compound-index of the other dataset)
+    # * c occurs only once in this dataset? map compound-index of other dataset to index in this dataset
+    # * c occurs >1 in this dataset?
+    # ** number of occurences is equal in both datasets? assume order is preserved(!) and map accordingly
+    # ** number of occurences is not equal in both datasets? cannot map, raise error
+    # @param [OpenTox::Dataset] dataset that should be mapped to this dataset (fully loaded)
+    # @param [Fixnum] compound_index, corresponding to dataset
+    def compound_index( dataset, compound_index )
+      unless defined?(@index_map) and @index_map[dataset.uri]
+        map = {}
+        dataset.compounds.collect{|c| c.uri}.uniq.each do |compound|
+          self_indices = compound_indices(compound)
+          next unless self_indices
+          dataset_indices = dataset.compound_indices(compound)
+          if self_indices.size==1  
+            dataset_indices.size.times do |i|
+              map[dataset_indices[i]] = self_indices[0]
+            end
+          elsif self_indices.size==dataset_indices.size
+            # we do assume that the order is preseverd!
+            dataset_indices.size.times do |i|
+              map[dataset_indices[i]] = self_indices[i]
+            end
+          else
+            raise "cannot map compound #{compound} from dataset #{dataset.uri} to dataset #{uri}, "+
+              "compound occurs #{dataset_indices.size} times and #{self_indices.size} times"
+          end
+        end  
+        @index_map = {} unless defined?(@index_map)
+        @index_map[dataset.uri] = map
+      end
+      @index_map[dataset.uri][compound_index]
+    end    
+    
+    def compound_indices( compound )
+      unless defined?(@cmp_indices) and @cmp_indices.has_key?(compound)
+        @cmp_indices = {}
+        @compounds.size.times do |i|
+          c = @compounds[i].uri
+          if @cmp_indices[c]==nil
+            @cmp_indices[c] = [i]
+          else
+            @cmp_indices[c] = @cmp_indices[c]+[i]
+          end   
+        end
+      end
+      @cmp_indices[compound]
+    end
+    
+    def data_entry_value(compound_index, feature_uri)
+      col = @features.collect{|f| f.uri}.index feature_uri
+      @data_entries[compound_index][col]
     end
   end
+
 end
diff --git a/lib/error.rb b/lib/error.rb
index 4fed343..eb72144 100644
--- a/lib/error.rb
+++ b/lib/error.rb
@@ -5,17 +5,19 @@ module OpenToxError
   attr_accessor :http_code, :uri
   def initialize message, uri=nil
     super message
-    @uri = uri.to_s.sub(%r{//.*:.*@},'//') # remove credentials from uri
-    @http_code ||= 500
-    @rdf = RDF::Graph.new
-    subject = RDF::Node.new
-    @rdf << [subject, RDF.type, RDF::OT.ErrorReport]
-    @rdf << [subject, RDF::OT.actor, @uri]
-    @rdf << [subject, RDF::OT.message, message.to_s]
-    @rdf << [subject, RDF::OT.statusCode, @http_code]
-    @rdf << [subject, RDF::OT.errorCode, self.class.to_s]
-    @rdf << [subject, RDF::OT.errorCause, short_backtrace]
-    $logger.error("\n"+self.to_turtle)
+    #unless self.is_a? Errno::EAGAIN # avoid "Resource temporarily unavailable" errors
+      @uri = uri.to_s.sub(%r{//.*:.*@},'//') # remove credentials from uri
+      @http_code ||= 500
+      @rdf = RDF::Graph.new
+      subject = RDF::Node.new
+      @rdf << [subject, RDF.type, RDF::OT.ErrorReport]
+      @rdf << [subject, RDF::OT.actor, @uri]
+      @rdf << [subject, RDF::OT.message, message.sub(/^"/,'').sub(/"$/,'')]
+      @rdf << [subject, RDF::OT.statusCode, @http_code]
+      @rdf << [subject, RDF::OT.errorCode, self.class.to_s]
+      @rdf << [subject, RDF::OT.errorCause, short_backtrace]
+      $logger.error("\n"+self.to_turtle)
+    #end
   end
 
   def short_backtrace
@@ -40,13 +42,14 @@ module OpenToxError
     prefixes = {:rdf => "http://www.w3.org/1999/02/22-rdf-syntax-ns#"}
     ['OT', 'DC', 'XSD', 'OLO'].each{|p| prefixes[p.downcase.to_sym] = eval("RDF::#{p}.to_s") }
     RDF::N3::Writer.for(:turtle).buffer(:prefixes => prefixes)  do |writer|
-      @rdf.each{|statement| writer << statement}
+      @rdf.each{|statement| writer << statement} if @rdf
     end
   end
 
 end
 
 class RuntimeError
+#class StandardError
   include OpenToxError
 end
 
@@ -58,6 +61,7 @@ end
 module OpenTox
 
   class Error < RuntimeError
+    include OpenToxError
     
     def initialize code, message, uri=nil
       @http_code = code
@@ -77,7 +81,7 @@ module OpenTox
     
     # define global methods for raising errors, eg. bad_request_error
     Object.send(:define_method, error[:method]) do |message,uri=nil|
-      raise c.new(message, uri)
+      raise c.new(message.inspect, uri)
     end
   end
   
diff --git a/lib/feature.rb b/lib/feature.rb
new file mode 100644
index 0000000..5d3d962
--- /dev/null
+++ b/lib/feature.rb
@@ -0,0 +1,37 @@
+module OpenTox
+
+  class Feature
+
+    # Find out feature type
+    # Classification takes precedence
+    # @return [String] Feature type
+    def feature_type
+      if self[RDF.type].include?(RDF::OT.NominalFeature)
+        "classification"
+      elsif self[RDF.type].include?(RDF::OT.NumericFeature)
+        "regression"
+      else
+        "unknown"
+      end
+    end
+
+    # Get accept values
+    # 
+    # @return[Array] Accept values
+    def accept_values
+      self[RDF::OT.acceptValue] ? self[RDF::OT.acceptValue].sort : nil
+    end
+
+    # Create value map
+    # @param [OpenTox::Feature] Feature
+    # @return [Hash] A hash with keys 1...feature.training_classes.size and values training classes
+    def value_map
+      unless defined? @value_map
+        accept_values ? @value_map = accept_values.each_index.inject({}) { |h,idx| h[idx+1]=accept_values[idx]; h } : @value_map = nil
+      end
+      @value_map
+    end
+
+  end
+
+end
diff --git a/lib/model.rb b/lib/model.rb
index c104e64..144c8c3 100644
--- a/lib/model.rb
+++ b/lib/model.rb
@@ -11,5 +11,36 @@ module OpenTox
       wait_for_task uri if wait
     end
 
+    def feature_type # CH: subjectid is a object variable, no need to pass it as a parameter
+      unless @feature_type
+        get unless metadata[OT.dependentVariables.to_s]
+        bad_request_error "Cannot determine feature type, dependent variable missing in model #{@uri}" unless metadata[OT.dependentVariables.to_s]
+        @feature_type = OpenTox::Feature.new( metadata[OT.dependentVariables.to_s][0], @subjectid ).feature_type
+      end
+      @feature_type
+    end
+    
+    def predicted_variable
+      load_predicted_variables unless defined? @predicted_variable
+      @predicted_variable
+    end
+    
+    def predicted_confidence
+      load_predicted_variables unless defined? @predicted_confidence
+      @predicted_confidence
+    end
+    
+    private
+    def load_predicted_variables
+      metadata[OT.predictedVariables.to_s].each do |f|
+        feat = OpenTox::Feature.find( f, @subjectid )
+        if feat.title =~ /confidence/
+          @predicted_confidence = f
+        else
+          @predicted_variable = f unless @predicted_variable
+        end 
+      end
+    end
+
   end
 end
diff --git a/lib/opentox-client.rb b/lib/opentox-client.rb
index 65a9177..1fe084b 100644
--- a/lib/opentox-client.rb
+++ b/lib/opentox-client.rb
@@ -34,15 +34,10 @@ FALSE_REGEXP = /^(false|inactive|0|0.0|low tox|deactivating|non-carcinogen|non-m
   "opentox.rb",
   "task.rb",
   "compound.rb",
+  "feature.rb",
   "dataset.rb",
   "model.rb",
   "algorithm.rb",
-  "4store.rb",
   "validation.rb"
-].each{ |f| require File.join(File.dirname(__FILE__),f) }
-
-Dir["#{File.dirname(__FILE__)}/utils/shims/*.rb"].each { |f| require f } # Shims for legacy code
-Dir["#{File.dirname(__FILE__)}/utils/sparql/*.rb"].each { |f| require f } # SPARQL code
-Dir["#{File.dirname(__FILE__)}/utils/rdf/*.rb"].each { |f| require f } # RDF code
-Dir["#{File.dirname(__FILE__)}/utils/*.rb"].each { |f| require f } # Utils for Libs
+].each{ |f| require_relative f }
 
diff --git a/lib/opentox.rb b/lib/opentox.rb
index 221a8dd..1251f33 100644
--- a/lib/opentox.rb
+++ b/lib/opentox.rb
@@ -3,109 +3,110 @@ $logger = OTLogger.new(STDERR)
 $logger.level = Logger::DEBUG
 
 module OpenTox
+  #include RDF CH: leads to namespace clashes with URI class
 
-  attr_accessor :uri, :subjectid, :rdf
+  attr_reader :uri, :subjectid
+  attr_writer :metadata, :parameters
 
   # Ruby interface
 
-  # Create a new OpenTox object (does not load data from service)
+  # Create a new OpenTox object 
   # @param [optional,String] URI
   # @param [optional,String] subjectid
   # @return [OpenTox] OpenTox object
   def initialize uri=nil, subjectid=nil
     @rdf = RDF::Graph.new
-    if uri
-      @uri = uri.to_s.chomp
-    else
-      service = self.class.to_s.split('::').last.downcase
-      service_uri = eval("$#{service}[:uri]")
-      bad_request_error "$#{service}[:uri] variable not set. Please set $#{service}[:uri] or use an explicit uri as first constructor argument " unless service_uri
-      @uri = File.join service_uri, SecureRandom.uuid
-    end
-    append RDF.type, eval("RDF::OT."+self.class.to_s.split('::').last)
-    append RDF::DC.date, DateTime.now
     @subjectid = subjectid
+    @metadata = {}
+    @parameters = []
+    uri ? @uri = uri.to_s.chomp : @uri = File.join(service_uri, SecureRandom.uuid)
   end
 
-  # Object metadata 
+  # Object metadata (lazy loading)
   # @return [Hash] Object metadata
-  def metadata 
-    # return plain strings instead of RDF objects
-    @rdf.to_hash[RDF::URI.new(@uri)].inject({}) { |h, (predicate, values)| h[predicate.to_s] = values.collect{|v| v.to_s}; h }
+  def metadata force_update=false
+    if (@metadata.empty? or force_update) and URI.accessible? @uri
+      get if @rdf.empty? or force_update 
+      # return values as plain strings instead of RDF objects
+      @metadata = @rdf.to_hash[RDF::URI.new(@uri)].inject({}) { |h, (predicate, values)| h[predicate] = values.collect{|v| v.to_s}; h }
+    end
+    @metadata
   end
 
   # Metadata values 
   # @param [String] Predicate URI
   # @return [Array, String] Predicate value(s)
   def [](predicate)
-    return nil if metadata[predicate.to_s].nil?
-    metadata[predicate.to_s].size == 1 ? metadata[predicate.to_s].first : metadata[predicate.to_s]
+    return nil if metadata[predicate].nil?
+    metadata[predicate].size == 1 ? metadata[predicate].first : metadata[predicate]
   end
 
-  # Set object metadata
+  # Set a metadata entry
   # @param [String] Predicate URI
   # @param [Array, String] Predicate value(s)
   def []=(predicate,values)
-    @rdf.delete [RDF::URI.new(@uri.to_s),RDF::URI.new(predicate.to_s),nil] 
-    append predicate.to_s, values
-  end
-
-  def parameters
-    params = {}
-    query = RDF::Query.new({
-      :parameter => {
-        RDF.type  => RDF::OT.Parameter,
-        :property => :value,
-      }
-    })
-    query.execute(@rdf).each do |solution|
-      params[solution.parameter] = {} unless params[solution.parameter] 
-      params[solution.parameter][solution.property.to_s] = solution.value.to_s
-    end
-    params.values
-  end
-
-  def parameters=(parameters)
-    parameters.each do |param|
-      p_node = RDF::Node.new
-      @rdf << [RDF::URI.new(@uri), RDF::OT.parameters, p_node]
-      @rdf << [p_node, RDF.type, RDF::OT.Parameter]
-      param.each{ |p,o| @rdf << [p_node, p, o] }
+    @metadata[predicate] = [values].flatten
+  end
+
+  def parameters force_update=false
+    if (@parameters.empty? or force_update) and URI.accessible? @uri
+      get if @rdf.empty? or force_update
+      params = {}
+      query = RDF::Query.new({
+        :parameter => {
+          RDF.type  => RDF::OT.Parameter,
+          :property => :value,
+        }
+      })
+      query.execute(@rdf).each do |solution|
+        params[solution.parameter] = {} unless params[solution.parameter] 
+        params[solution.parameter][solution.property] = solution.value
+      end
+      @parameters = params.values
     end
+    @parameters
   end
 
-  # Append object metadata
-  # @param [String] Predicate URI
-  # @param [Array, String] Predicate value(s)
-  def append(predicate,values)
-    uri = RDF::URI.new @uri
-    predicate = RDF::URI.new predicate
-    [values].flatten.each { |value| @rdf << [uri, predicate, value] }
+  def parameter_value title
+    @parameters.collect{|p| p[RDF::OT.paramValue] if p[RDF::DC.title] == title}.compact.first
   end
 
   # Get object from webservice
   def get mime_type="text/plain"
+    bad_request_error "Mime type #{mime_type} is not supported. Please use 'text/plain' (default) or 'application/rdf+xml'." unless mime_type == "text/plain" or mime_type == "application/rdf+xml"
     response = RestClientWrapper.get(@uri,{},{:accept => mime_type, :subjectid => @subjectid})
     if URI.task?(response)
-      wait_for_task response
-      response = RestClientWrapper.get(t.resultURI,{},{:accept => mime_type, :subjectid => @subjectid})
+      uri = wait_for_task response
+      response = RestClientWrapper.get(uri,{},{:accept => mime_type, :subjectid => @subjectid})
     end
     parse_ntriples response if mime_type == "text/plain"
     parse_rdfxml response if mime_type == "application/rdf+xml"
   end
 
-  # Post object to webservice
-  def post params=nil, wait=true
-    # TODO: RDFXML
-    uri = RestClientWrapper.post @uri.to_s, params, { :content_type => "text/plain", :subjectid => @subjectid}
-    wait_for_task uri if wait
-  end
-
-  # Save object at webservice
-  def put wait=true
-    # TODO: RDFXML
-    uri = RestClientWrapper.put @uri.to_s, self.to_ntriples, { :content_type => "text/plain", :subjectid => @subjectid}
-    wait_for_task uri if wait
+  # Post object to webservice (append to object), rarely useful and deprecated 
+  def post wait=true, mime_type="text/plain"
+    bad_request_error "Mime type #{mime_type} is not supported. Please use 'text/plain' (default) or 'application/rdf+xml'." unless mime_type == "text/plain" or mime_type == "application/rdf+xml"
+    case mime_type
+    when 'text/plain'
+      body = self.to_ntriples
+    when 'application/rdf+xml'
+      body = self.to_rdfxml
+    end
+    uri = RestClientWrapper.post @uri.to_s, body, { :content_type => mime_type, :subjectid => @subjectid}
+    wait ? wait_for_task(uri) : uri
+  end
+
+  # Save object at webservice (replace or create object)
+  def put wait=true, mime_type="text/plain"
+    bad_request_error "Mime type #{mime_type} is not supported. Please use 'text/plain' (default) or 'application/rdf+xml'." unless mime_type == "text/plain" or mime_type == "application/rdf+xml"
+    case mime_type
+    when 'text/plain'
+      body = self.to_ntriples
+    when 'application/rdf+xml'
+      body = self.to_rdfxml
+    end
+    uri = RestClientWrapper.put @uri.to_s, body, { :content_type => mime_type, :subjectid => @subjectid}
+    wait ? wait_for_task(uri) : uri
   end
 
   # Delete object at webservice
@@ -113,29 +114,30 @@ module OpenTox
     RestClientWrapper.delete(@uri.to_s,nil,{:subjectid => @subjectid})
   end
 
-  def wait_for_task uri
-    OpenTox.wait_for_task uri
+  def service_uri
+    self.class.service_uri
   end
 
-  def self.wait_for_task uri
-    if URI.task?(uri) 
-      t = OpenTox::Task.new uri
-      t.wait
-      unless t.completed?
-        method = RestClientWrapper.known_errors.select{|error| error[:code] == t.code}.first[:method]
-        t.get
-        Object.send(method,t.error_report[RDF::OT.message],t.uri)
-      end
-      uri = t.resultURI
+  def create_rdf
+    @rdf = RDF::Graph.new
+    @metadata[RDF.type] ||= eval("RDF::OT."+self.class.to_s.split('::').last)
+    @metadata[RDF::DC.date] ||= DateTime.now
+    @metadata.each do |predicate,values|
+      [values].flatten.each { |value| @rdf << [RDF::URI.new(@uri), predicate, value] }
+    end
+    @parameters.each do |parameter|
+      p_node = RDF::Node.new
+      @rdf << [RDF::URI.new(@uri), RDF::OT.parameters, p_node]
+      @rdf << [p_node, RDF.type, RDF::OT.Parameter]
+      parameter.each { |k,v| @rdf << [p_node, k, v] }
     end
-    uri
   end
 
   RDF_FORMATS.each do |format|
 
     # rdf parse methods for all formats e.g. parse_rdfxml
-    send :define_method, "parse_#{format}".to_sym do |rdf,init=true|
-      @rdf = RDF::Graph.new if init
+    send :define_method, "parse_#{format}".to_sym do |rdf|
+      @rdf = RDF::Graph.new
       RDF::Reader.for(format).new(rdf) do |reader|
         reader.each_statement{ |statement| @rdf << statement }
       end
@@ -143,25 +145,27 @@ module OpenTox
 
     # rdf serialization methods for all formats e.g. to_rdfxml
     send :define_method, "to_#{format}".to_sym do
-      RDF::Writer.for(format).buffer do |writer|
+      create_rdf
+      RDF::Writer.for(format).buffer(:encoding => Encoding::ASCII) do |writer|
         @rdf.each{|statement| writer << statement}
       end
     end
   end
 
-  def to_turtle # redefine to use prefixes (not supported by RDF::Writer)
+  def to_turtle # redefined to use prefixes (not supported by RDF::Writer)
     prefixes = {:rdf => "http://www.w3.org/1999/02/22-rdf-syntax-ns#"}
     ['OT', 'DC', 'XSD', 'OLO'].each{|p| prefixes[p.downcase.to_sym] = eval("RDF::#{p}.to_s") }
+    create_rdf
     RDF::N3::Writer.for(:turtle).buffer(:prefixes => prefixes)  do |writer|
       @rdf.each{|statement| writer << statement}
     end
   end
 
-  {
-    :title => RDF::DC.title,
-    :dexcription => RDF::DC.description,
-    :type => RDF.type
-  }.each do |method,predicate|
+  def to_html
+    to_turtle.to_html
+  end
+
+  { :title => RDF::DC.title, :dexcription => RDF::DC.description, :type => RDF.type }.each do |method,predicate|
     send :define_method, method do 
       self.[](predicate) 
     end
@@ -170,15 +174,58 @@ module OpenTox
     end
   end
 
-  # create default OpenTox classes
+  # create default OpenTox classes with class methods
   CLASSES.each do |klass|
     c = Class.new do
       include OpenTox
 
-      def self.all service_uri, subjectid=nil
+      def self.all subjectid=nil
         uris = RestClientWrapper.get(service_uri, {}, :accept => 'text/uri-list').split("\n").compact
         uris.collect{|uri| self.new(uri, subjectid)}
       end
+
+      def self.find uri, subjectid=nil
+        URI.accessible?(uri) ? self.new(uri, subjectid) : nil
+      end
+
+      def self.create metadata, subjectid=nil 
+        object = self.new nil, subjectid
+        object.metadata = metadata
+        object.put
+        object
+      end
+
+      def self.find_or_create metadata, subjectid=nil
+        sparql = "SELECT DISTINCT ?s WHERE { "
+        metadata.each do |predicate,objects|
+          unless [RDF::DC.date,RDF::DC.modified,RDF::DC.description].include? predicate # remove dates and description (strange characters in description may lead to SPARQL errors)
+            if objects.is_a? String
+              URI.valid?(objects) ? o = "<#{objects}>" : o = "'''#{objects}'''" 
+              sparql << "?s <#{predicate}> #{o}. " 
+            elsif objects.is_a? Array
+              objects.each do |object|
+                URI.valid?(object) ? o = "<#{object}>" : o = "'#{object}'" 
+                sparql << "?s <#{predicate}> #{o}. " 
+              end
+            end
+          end
+        end
+        sparql <<  "}"
+        uris = RestClientWrapper.get(service_uri,{:query => sparql},{:accept => "text/uri-list", :subjectid => @subjectid}).split("\n")
+        if uris.empty?
+          self.create metadata, subjectid
+        else
+          self.new uris.first
+        end
+      end
+
+      def self.service_uri
+        service = self.to_s.split('::').last.downcase
+        eval("$#{service}[:uri]")
+      rescue
+        bad_request_error "$#{service}[:uri] variable not set. Please set $#{service}[:uri] or use an explicit uri as first constructor argument "
+      end
+
     end
     OpenTox.const_set klass,c
   end
diff --git a/lib/overwrite.rb b/lib/overwrite.rb
index c7bb312..d27434b 100644
--- a/lib/overwrite.rb
+++ b/lib/overwrite.rb
@@ -1,3 +1,4 @@
+require "base64"
 class Object
   # An object is blank if it's false, empty, or a whitespace string.
   # For example, "", "   ", +nil+, [], and {} are all blank.
@@ -17,6 +18,7 @@ module Enumerable
 end
 
 class String
+
   def underscore
     self.gsub(/::/, '/').
     gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
@@ -24,6 +26,44 @@ class String
     tr("-", "_").
     downcase
   end
+
+  # encloses URI in text with with link tag
+  # @return [String] new text with marked links
+  def link_urls
+    self.gsub(/(?i)http(s?):\/\/[^\r\n\s']*/, '<a href="\0">\0</a>')
+  end
+
+  # produces a html page for making web services browser friendly
+  # format of text (=string params) is preserved (e.g. line breaks)
+  # urls are marked as links
+  #
+  # @param [String] text this is the actual content, 
+  # @param [optional,String] related_links info on related resources
+  # @param [optional,String] description general info
+  # @param [optional,Array] post_command, infos for the post operation, object defined below
+  # @return [String] html page
+  def to_html(related_links=nil, description=nil, png_image=nil  )
+    
+    # TODO add title as parameter
+    title = nil #$sinatra.to($sinatra.request.env['PATH_INFO'], :full) if $sinatra
+    html = "<html>"
+    html << "<title>"+title+"</title>" if title
+    #html += "<img src=\""+OT_LOGO+"\"><\/img><body>"
+      
+    html << "<h3>Description</h3><pre><p>"+description.link_urls+"</p></pre>" if description
+    html << "<h3>Related links</h3><pre><p>"+related_links.link_urls+"</p></pre>" if related_links
+    html << "<h3>Content</h3>" if description || related_links
+    html << "<pre><p style=\"padding:15px; border:10px solid \#B9DCFF\">"
+    html << "<img src=\"data:image/png;base64,#{Base64.encode64(png_image)}\">\n" if png_image
+    html << self.link_urls
+    html << "</p></pre></body></html>"
+    html
+  end
+  
+  def uri?
+    URI.valid?(self)
+  end
+
 end
 
 module URI
@@ -84,11 +124,11 @@ class File
   end
 end
 
-# overwrite backtick operator to catch system errors
 module Kernel
 
-  # Override raises an error if _cmd_ returns a non-zero exit status.
-  # Returns stdout if _cmd_ succeeds.  Note that these are simply concatenated; STDERR is not inline.
+  # overwrite backtick operator to catch system errors
+  # Override raises an error if _cmd_ returns a non-zero exit status. CH: I do not understand this comment
+  # Returns stdout if _cmd_ succeeds.  Note that these are simply concatenated; STDERR is not inline. CH: I do not understand this comment
   def ` cmd
     stdout, stderr = ''
     status = Open4::popen4(cmd) do |pid, stdin_stream, stdout_stream, stderr_stream|
@@ -101,6 +141,27 @@ module Kernel
     internal_server_error $!.message 
   end
 
+  def wait_for_task uri
+    if URI.task?(uri) 
+      t = OpenTox::Task.new uri
+      t.wait
+      unless t.completed?
+        begin # handle known (i.e. OpenTox) errors
+          error = OpenTox::RestClientWrapper.known_errors.select{|error| error[:code] == t.code}.first
+          error ? error_method = error[:method] : error_method = :internal_server_error
+          report = t.error_report
+          report ? error_message = report[RDF::OT.message] : error_message = $!.message
+          Object.send(error_method,error_message,t.uri)
+        rescue
+          internal_server_error "#{$!.message}\n#{$!.backtrace}", t.uri
+        end
+      end
+      uri = t.resultURI
+    end
+    uri
+  end
+
+
 end
 
 
@@ -126,6 +187,5 @@ class Array
     return self.uniq.size == 1
   end
 
-
 end
 
diff --git a/lib/rest-client-wrapper.rb b/lib/rest-client-wrapper.rb
index f3c56c8..38219c1 100644
--- a/lib/rest-client-wrapper.rb
+++ b/lib/rest-client-wrapper.rb
@@ -37,8 +37,6 @@ module OpenTox
         headers.each{ |k,v| headers.delete(k) if v==nil } if headers #remove keys with empty values, as this can cause problems
         args[:headers] = headers 
         
-        $logger.debug "POST #{uri} #{payload.inspect}" if method.to_s=="post" && payload.is_a?(Hash)
-
         @request = RestClient::Request.new(args)
         # ignore error codes from Task services (may return error codes >= 400 according to API, which causes exceptions in RestClient and RDF::Reader)
         @response = @request.execute do |response, request, result|
diff --git a/lib/task.rb b/lib/task.rb
index 82e1665..07c0b05 100644
--- a/lib/task.rb
+++ b/lib/task.rb
@@ -6,33 +6,55 @@ module OpenTox
 
     attr_accessor :pid, :observer_pid
 
-    def self.create service_uri, subjectid=nil, params={}
+    def metadata
+      super true # always update metadata
+    end
+
+    def self.run(description, creator=nil, subjectid=nil)
 
-      uri = File.join(service_uri,SecureRandom.uuid)
-      task = Task.new uri, subjectid
+      task = Task.new nil, subjectid
       task[RDF::OT.created_at] = DateTime.now
       task[RDF::OT.hasStatus] = "Running"
-      params.each { |k,v| task[k] = v }
-      task.put false
+      task[RDF::DC.description] = description.to_s
+      task[RDF::DC.creator] = creator.to_s
+      task.put
       pid = fork do
         begin
-          result_uri = yield 
-          task.completed result_uri
+          task.completed yield
+          #result_uri = yield 
+          #task.completed result_uri
         rescue
-          unless $!.is_a?(RuntimeError) # PENDING: only runtime Errors are logged when raised
-            cut_index = $!.backtrace.find_index{|line| line.match /gems\/sinatra/}
-            cut_index = -1 unless cut_index
+=begin
+          #unless $!.is_a?(RuntimeError) # PENDING: only runtime Errors are logged when raised
             msg = "\nTask ERROR\n"+
-              "task description: #{params[RDF::DC.description]}\n"+
+              "task description: #{task[RDF::DC.description]}\n"+
               "task uri:         #{$!.class.to_s}\n"+
               "error msg:        #{$!.message}\n"+
               "error backtrace:\n#{$!.backtrace[0..cut_index].join("\n")}\n"
             $logger.error msg
-          end
+          #end
+=end
           if $!.respond_to? :to_ntriples
+            puts $!.to_turtle
             RestClientWrapper.put(File.join(task.uri,'Error'),:errorReport => $!.to_ntriples,:content_type => 'text/plain') 
           else
-            RestClientWrapper.put(File.join(task.uri,'Error')) 
+            cut_index = $!.backtrace.find_index{|line| line.match /gems\/sinatra/}
+            cut_index = -1 unless cut_index
+            @rdf = RDF::Graph.new
+            subject = RDF::Node.new
+            @rdf << [subject, RDF.type, RDF::OT.ErrorReport]
+            @rdf << [subject, RDF::OT.message, $!.message]
+            @rdf << [subject, RDF::OT.errorCode, $!.class.to_s]
+            @rdf << [subject, RDF::OT.errorCause, $!.backtrace[0..cut_index].join("\n")]
+            prefixes = {:rdf => "http://www.w3.org/1999/02/22-rdf-syntax-ns#", :ot => RDF::OT.to_s}
+            turtle = RDF::N3::Writer.for(:turtle).buffer(:prefixes => prefixes)  do |writer|
+              @rdf.each{|statement| writer << statement}
+            end
+            $logger.error turtle
+            nt = RDF::Writer.for(:ntriples).buffer do |writer|
+              @rdf.each{|statement| writer << statement}
+            end
+            RestClientWrapper.put(File.join(task.uri,'Error'),:errorReport => nt,:content_type => 'text/plain') 
           end
           task.kill
         end
@@ -73,23 +95,21 @@ module OpenTox
       kill
       self.[]=(RDF::OT.hasStatus, "Cancelled")
       self.[]=(RDF::OT.finished_at, DateTime.now)
-      put false
+      put
     end
 
     def completed(uri)
       self.[]=(RDF::OT.resultURI, uri)
       self.[]=(RDF::OT.hasStatus, "Completed")
       self.[]=(RDF::OT.finished_at, DateTime.now)
-      put false
+      put
     end
 
     # waits for a task, unless time exceeds or state is no longer running
-    # @param [optional,Numeric] dur seconds pausing before checking again for completion
-    # TODO: add waiting task
     def wait
       start_time = Time.new
       due_to_time = start_time + DEFAULT_TASK_MAX_DURATION
-      dur = 0.3
+      dur = 0.2
       while running? 
         sleep dur
         dur = [[(Time.new - start_time)/20.0,0.3].max,300.0].min
@@ -122,14 +142,20 @@ module OpenTox
 
   [:hasStatus, :resultURI, :created_at, :finished_at].each do |method|
     define_method method do
-      get
       response = self.[](RDF::OT[method])
       response = self.[](RDF::OT1[method]) unless response  # API 1.1 compatibility
       response
     end
   end
 
+  # Check status of a task
+  # @return [String] Status
+  def status
+    self[RDF::OT.hasStatus]
+  end
+
   def error_report
+    get
     report = {}
     query = RDF::Query.new({
       :report => {
@@ -144,5 +170,31 @@ module OpenTox
   end
 
   #TODO: subtasks (only for progress)
+  class SubTask
+    
+    def initialize(task, min, max)
+      #TODO add subtask code
+    end
+
+    def self.create(task, min, max)
+      if task
+        SubTask.new(task, min, max)
+      else
+        nil
+      end
+    end
+    
+    def waiting_for(task_uri)
+      #TODO add subtask code
+    end
+    
+    def progress(pct)
+      #TODO add subtask code
+    end
+    
+    def running?()
+      #TODO add subtask code
+    end
+  end
 
 end
diff --git a/lib/utils/diag.rb b/lib/utils/diag.rb
deleted file mode 100644
index fd37945..0000000
--- a/lib/utils/diag.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-=begin
-* Name: diag.rb
-* Description: Diagnostic tools
-* Author: Andreas Maunz <andreas@maunz.de>
-* Date: 10/2012
-=end
-
-# Print a diagnostic message
-def uri_list 
-  puts "My load path is:\n#{$LOAD_PATH.join("\n")} \nI have loaded #{$LOADED_FEATURES.size} objects.\n\n"
-end
diff --git a/lib/utils/html.rb b/lib/utils/html.rb
deleted file mode 100644
index 4a4c9e1..0000000
--- a/lib/utils/html.rb
+++ /dev/null
@@ -1,52 +0,0 @@
-#OT_LOGO = File.join(CONFIG[:services]["opentox-validation"],"resources/ot-logo.png")
-
-=begin
-* Name: html.rb
-* Description: Tools to provide html output
-* Author: Andreas Maunz <andreas@maunz.de>
-* Date: 10/2012
-=end
-
-require "base64"
-
-# AM: needed since this gem has a nested directory structure
-
-class String
-  # encloses URI in text with with link tag
-  # @return [String] new text with marked links
-  def link_urls
-    self.gsub(/(?i)http(s?):\/\/[^\r\n\s']*/, '<a href="\0">\0</a>')
-  end
-  
-end
-
-module OpenTox
-  
-  # produces a html page for making web services browser friendly
-  # format of text (=string params) is preserved (e.g. line breaks)
-  # urls are marked as links
-  #
-  # @param [String] text this is the actual content, 
-  # @param [optional,String] related_links info on related resources
-  # @param [optional,String] description general info
-  # @param [optional,Array] post_command, infos for the post operation, object defined below
-  # @return [String] html page
-  def self.text_to_html( text, subjectid=nil, related_links=nil, description=nil, post_command=nil, png_image=nil  )
-    
-    # TODO add title as parameter
-    title = nil #$sinatra.to($sinatra.request.env['PATH_INFO'], :full) if $sinatra
-    html = "<html>"
-    html += "<title>"+title+"</title>" if title
-    #html += "<img src=\""+OT_LOGO+"\"><\/img><body>"
-      
-    html += "<h3>Description</h3><pre><p>"+description.link_urls+"</p></pre>" if description
-    html += "<h3>Related links</h3><pre><p>"+related_links.link_urls+"</p></pre>" if related_links
-    html += "<h3>Content</h3>" if description || related_links
-    html += "<pre><p style=\"padding:15px; border:10px solid \#B9DCFF\">"
-    html += "<img src=\"data:image/png;base64,#{Base64.encode64(png_image)}\">\n" if png_image
-    html += text.link_urls
-    html += "</p></pre></body></html>"
-    html
-  end
-  
-end
diff --git a/lib/utils/rdf/dataset.rb b/lib/utils/rdf/dataset.rb
deleted file mode 100644
index 2cb32a9..0000000
--- a/lib/utils/rdf/dataset.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-=begin
-* Name: dataset.rb
-* Description: Dataset RDF tools
-* Author: Andreas Maunz <andreas@maunz.de>
-* Date: 10/2012
-=end
-
-module OpenTox
-  class Dataset
-
-    # Load features via RDF (slow)
-    # @param [String] uri Dataset URI
-    # @return [Array] features Features in order
-    def find_features_rdf
-      query = RDF::Query.new do
-        pattern [:uri, RDF.type, RDF::OT.Feature]
-        pattern [:uri, RDF::OLO.index, :idx]
-      end
-      query.execute(@rdf).sort_by{|s| s.idx}.collect{|s| OpenTox::Feature.new(s.uri.to_s)}
-    end
-
-    # Load compounds via RDF (slow)
-    # @param [String] uri Dataset URI
-    # @return [Array] compounds Compounds in order
-    def find_compounds_rdf
-      query = RDF::Query.new do
-        pattern [:uri, RDF.type, RDF::OT.Compound]
-        pattern [:uri, RDF::OLO.index, :idx]
-      end
-      query.execute(@rdf).sort_by{|s| s.idx}.collect{|s| OpenTox::Compound.new(s.uri.to_s)}
-    end
-
-    # Load data entries via RDF (slow)
-    # @param [String] uri Dataset uri
-    # @return [Array] entries Data entries, ordered primarily over cols and secondarily over rows
-    def find_data_entries_rdf
-      query = RDF::Query.new do
-        pattern [:data_entry, RDF::OLO.index, :cidx] # compound index: now a free variable
-        pattern [:data_entry, RDF::OT.values, :vals]
-        pattern [:vals, RDF::OT.feature, :f]
-        pattern [:f, RDF::OLO.index, :fidx]
-        pattern [:vals, RDF::OT.value, :val]
-      end
-      query.execute(@rdf).order_by(:fidx, :cidx).collect { |s| s.val.to_s }
-    end
-
-    # Query a dataset URI for ordered status 
-    # by loading its metadata (OpenTox compliant)
-    # @param [String] uri Dataset uri
-    # @return [TrueClass, FalseClass] status Whether the dataset is ordered
-    def self.ordered?(uri)
-      ds = OpenTox::Dataset.new # dummy 
-      ds.parse_rdfxml(RestClient.get([uri,"metadata"].join("/"),{:accept => "application/rdf+xml"}))
-      query = RDF::Query.new do
-        pattern [:dataset, RDF.type, RDF::OT.OrderedDataset]
-      end
-      query.execute(ds.rdf).size>0
-    end
-
-    # Load dataset URI from given RDF (slow)
-    # @param [String] rdf RDF
-    # @return [String] uri URI
-    def self.uri_from_rdf(rdf)
-      ds = OpenTox::Dataset.new # dummy 
-      ds.parse_rdfxml(rdf)
-      query = RDF::Query.new do
-        pattern [:dataset, RDF.type, RDF::OT.Dataset]
-      end
-      query.execute(ds.rdf).collect { |s| s.dataset.to_s }[0]
-    end
-
-  end
-end
diff --git a/lib/utils/shims/dataset.rb b/lib/utils/shims/dataset.rb
deleted file mode 100644
index f72ff1b..0000000
--- a/lib/utils/shims/dataset.rb
+++ /dev/null
@@ -1,201 +0,0 @@
-=begin
-* Name: dataset.rb
-* Description: Dataset shims
-* Author: Andreas Maunz <andreas@maunz.de>
-* Date: 10/2012
-=end
-
-module OpenTox
-
-  # Shims for the Dataset Class
-  class Dataset
-
-    attr_accessor :feature_positions, :compound_positions
-
-    # Load a dataset from URI
-    # @param [String] Dataset URI
-    # @return [OpenTox::Dataset] Dataset object
-    def self.find(uri, subjectid=nil)
-      return nil unless uri
-      ds = OpenTox::Dataset.new uri, subjectid
-      ds.get
-      ds
-    end
-    
-    def self.exist?(uri, subjectid=nil)
-      ds = OpenTox::Dataset.new uri, subjectid
-      begin
-        ds.get_metadata
-        true
-      rescue
-        false
-      end
-    end
-    
-    def split( compound_indices, feats, metadata, subjectid=nil)
-      
-      raise "Dataset.split : pls give compounds as indices" if compound_indices.size==0 or !compound_indices[0].is_a?(Fixnum)
-      raise "Dataset.split : pls give features as feature objects (given: #{feats})" if feats!=nil and feats.size>0 and !feats[0].is_a?(OpenTox::Feature)
-      $logger.debug "split dataset using "+compound_indices.size.to_s+"/"+@compounds.size.to_s+" compounds"
-      
-      dataset = OpenTox::Dataset.new(nil, subjectid)
-      dataset.metadata = metadata
-      dataset.features = (feats ? feats : self.features)
-      compound_indices.each do |c_idx|
-        dataset << [ self.compounds[c_idx] ] + dataset.features.each_with_index.collect{|f,f_idx| self.data_entries[c_idx][f_idx]} 
-      end
-
-      #compound_indices.each do |c_idx|
-        # c = @compounds[c_idx]
-        # dataset.add_compound(c)
-        # if @data_entries[c]
-          # features.each do |f|
-            # if @data_entries[c][f] 
-              # dataset.add_data_entry c,f,@data_entries[c][f][entry_index(c_idx)]
-            # else
-              # dataset.add_data_entry c,f,nil
-            # end
-          # end
-        # end
-      # end
-      
-      dataset.put subjectid
-      dataset    
-    end
-    
-
-    # maps a compound-index from another dataset to a compound-index from this dataset
-    # mapping works as follows:
-    # (compound c is the compound identified by the compound-index of the other dataset)
-    # * c occurs only once in this dataset? map compound-index of other dataset to index in this dataset
-    # * c occurs >1 in this dataset?
-    # ** number of occurences is equal in both datasets? assume order is preserved(!) and map accordingly
-    # ** number of occurences is not equal in both datasets? cannot map, raise error
-    # @param [OpenTox::Dataset] dataset that should be mapped to this dataset (fully loaded)
-    # @param [Fixnum] compound_index, corresponding to dataset
-    def compound_index( dataset, compound_index )
-      unless defined?(@index_map) and @index_map[dataset.uri]
-        map = {}
-        dataset.compounds.collect{|c| c.uri}.uniq.each do |compound|
-          self_indices = compound_indices(compound)
-          next unless self_indices
-          dataset_indices = dataset.compound_indices(compound)
-          if self_indices.size==1  
-            dataset_indices.size.times do |i|
-              map[dataset_indices[i]] = self_indices[0]
-            end
-          elsif self_indices.size==dataset_indices.size
-            # we do assume that the order is preseverd!
-            dataset_indices.size.times do |i|
-              map[dataset_indices[i]] = self_indices[i]
-            end
-          else
-            raise "cannot map compound #{compound} from dataset #{dataset.uri} to dataset #{uri}, "+
-              "compound occurs #{dataset_indices.size} times and #{self_indices.size} times"
-          end
-        end  
-        @index_map = {} unless defined?(@index_map)
-        @index_map[dataset.uri] = map
-      end
-      @index_map[dataset.uri][compound_index]
-    end    
-    
-    def compound_indices( compound )
-      unless defined?(@cmp_indices) and @cmp_indices.has_key?(compound)
-        @cmp_indices = {}
-        @compounds.size.times do |i|
-          c = @compounds[i].uri
-          if @cmp_indices[c]==nil
-            @cmp_indices[c] = [i]
-          else
-            @cmp_indices[c] = @cmp_indices[c]+[i]
-          end   
-        end
-      end
-      @cmp_indices[compound]
-    end
-    
-    def data_entry_value(compound_index, feature_uri)
-      build_feature_positions unless @feature_positions
-      @data_entries[compound_index][@feature_positions[feature_uri]]
-    end
-
-    ### Index Structures
-
-    # Create value map
-    # @param [OpenTox::Feature] A feature
-    # @return [Hash] A hash with keys 1...feature.training_classes.size and values training classes
-    def value_map(feature)
-      training_classes = feature.accept_values
-      raise "no accept values for feature #{feature.uri} in dataset #{uri}" unless training_classes
-      training_classes.each_index.inject({}) { |h,idx| h[idx+1]=training_classes[idx]; h } 
-    end
-
-    # Create feature positions map
-    # @return [Hash] A hash with keys feature uris and values feature positions
-    def build_feature_positions
-      unless @feature_positions
-        @feature_positions = @features.each_index.inject({}) { |h,idx| 
-          internal_server_error "Duplicate Feature '#{@features[idx].uri}' in dataset '#{@uri}'" if h[@features[idx].uri]
-          h[@features[idx].uri] = idx 
-          h
-        }
-      end
-    end
-
-    # Create compounds positions map
-    # @return [Hash] A hash with keys compound uris and values compound position arrays
-    def build_compound_positions
-      unless @compound_positions
-        @compound_positions = @compounds.each_index.inject({}) { |h,idx| 
-          inchi=OpenTox::Compound.new(@compounds[idx].uri).inchi
-          h[inchi] = [] unless h[inchi]
-          h[inchi] << idx if inchi =~ /InChI/
-          h
-        }
-      end
-    end
-
-
-    ### Associative Search Operations
-
-    # Search a dataset for a feature given its URI
-    # @param [String] Feature URI
-    # @return [OpenTox::Feature] Feature object, or nil if not present
-    def find_feature(uri)
-      build_feature_positions
-      res = @features[@feature_positions[uri]] if @feature_positions[uri]
-      res
-    end
-
-    # Search a dataset for a compound given its URI
-    # @param [String] Compound URI
-    # @return [OpenTox::Compound] Array of compound objects, or nil if not present
-    def find_compound(uri)
-      build_compound_positions
-      inchi = OpenTox::Compound.new(uri).inchi
-      res = @compounds[@compound_positions[inchi]] if inchi =~ /InChI/ and @compound_positions[inchi]
-      res
-    end
-
-    # Search a dataset for a data entry given compound URI and feature URI
-    # @param [String] Compound URI
-    # @param [String] Feature URI
-    # @return [Object] Data entry, or nil if not present
-    def find_data_entry(compound_uri, feature_uri)
-      build_compound_positions
-      build_feature_positions
-      inchi = OpenTox::Compound.new(compound_uri).inchi
-      if @compound_positions[inchi] && @feature_positions[feature_uri]
-        res = []
-        @compound_positions[inchi].each { |idx|
-          res << data_entries[idx][@feature_positions[feature_uri]]
-        }
-      end
-      res
-    end
-
-  end
-
-
-end
diff --git a/lib/utils/shims/feature.rb b/lib/utils/shims/feature.rb
deleted file mode 100644
index 9afa5c2..0000000
--- a/lib/utils/shims/feature.rb
+++ /dev/null
@@ -1,87 +0,0 @@
-=begin
-* Name: feature.rb
-* Description: Feature shims
-* Author: Andreas Maunz <andreas@maunz.de>
-* Date: 10/2012
-=end
-
-module OpenTox
-
-  # Shims for the feature class
-  class Feature
-
-    # Load a feature from URI
-    # @param [String] Feature URI
-    # @return [OpenTox::Feature] Feature object with the full data
-    def self.find(uri, subjectid=nil)
-      return nil unless uri
-      f = OpenTox::Feature.new uri, subjectid
-      f.get
-      f
-    end
-
-    # Load or create a feature given its title and metadata
-    # Create it if: a) not present, or b) present, but differs in metadata
-    # Newly created features are stored at the backend
-    # @param[String] title Feature title
-    # @param[Hash] metadata Feature metadata
-    # @return [OpenTox::Feature] Feature object with the full data, or nil
-    def self.find_by_title(title, metadata)
-      metadata[RDF.type] = [] unless metadata[RDF.type]
-      metadata[RDF.type] << RDF::OT.Feature unless metadata[RDF.type].include?(RDF::OT.Feature)
-      metadata[RDF::DC.title] = title unless (metadata[RDF::DC.title])
-      feature = feature_new = OpenTox::Feature.new(File.join($feature[:uri], SecureRandom.uuid), @subjectid)
-      feature_new.metadata = metadata
-      sparql = "SELECT DISTINCT ?feature WHERE { ?feature <#{RDF.type}> <#{RDF::OT['feature'.capitalize]}>. ?feature <#{RDF::DC.title}> '#{title.to_s}' }"
-      feature_uris = OpenTox::Backend::FourStore.query(sparql,"text/uri-list").split("\n")
-      features_equal = false # relevant also when no features found
-      feature_uris.each_with_index { |feature_uri,idx|
-        feature_existing = OpenTox::Feature.find(feature_uri, @subjectid)
-        if (feature_new.metadata.size+1 == feature_existing.metadata.size) # +1 due to title
-          features_equal = metadata.keys.collect { |predicate|
-            unless ( predicate == RDF::DC.title )
-              if feature_new[predicate].class == feature_existing[predicate].class
-                case feature_new[predicate].class.to_s
-                  when "Array" then (feature_new[predicate].sort == feature_existing[predicate].sort)
-                  else (feature_new[predicate] == feature_existing[predicate])
-                end
-              end
-            else
-              true
-            end
-          }.uniq == [true]
-        end
-        (feature=feature_existing and break) if features_equal
-      }
-      unless features_equal
-        feature_new.put 
-      end
-      feature
-    end
-
-    # Find out feature type
-    # Classification takes precedence
-    # @return [String] Feature type
-    def feature_type
-      bad_request_error "rdf type of feature '#{@uri}' not set" unless self[RDF.type]
-      if self[RDF.type].include?(OT.NominalFeature)
-        "classification"
-      elsif self[RDF.type].include?(OT.NumericFeature)
-        "regression"
-      else
-        "unknown"
-      end
-    end
-
-    # Get accept values
-    # @param[String] Feature URI
-    # @return[Array] Accept values
-    def accept_values
-      accept_values = self[OT.acceptValue]
-      accept_values.sort if accept_values
-      accept_values
-    end
-
-  end
-
-end
diff --git a/lib/utils/shims/model.rb b/lib/utils/shims/model.rb
deleted file mode 100644
index 26a82c4..0000000
--- a/lib/utils/shims/model.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-
-
-module OpenTox
-
-  # Shims for the Task class
-  class Model
-
-    def feature_type(subjectid=nil)
-      unless @feature_type
-        get unless metadata[OT.dependentVariables.to_s]
-        raise "cannot determine feature type, dependent variable missing" unless metadata[OT.dependentVariables.to_s]
-        @feature_type = OpenTox::Feature.find( metadata[OT.dependentVariables.to_s][0], subjectid ).feature_type
-      end
-      @feature_type
-    end
-    
-    def predicted_variable(subjectid=nil)
-      load_predicted_variables(subjectid) unless defined? @predicted_var 
-      @predicted_var
-    end
-    
-    def predicted_confidence(subjectid=nil)
-      load_predicted_variables(subjectid) unless defined? @predicted_conf 
-      @predicted_conf
-    end
-    
-    private
-    def load_predicted_variables(subjectid=nil)
-      metadata[OT.predictedVariables.to_s].each do |f|
-        feat = OpenTox::Feature.find( f, subjectid )
-        if feat.title =~ /confidence/
-          @predicted_conf = f
-        else
-          @predicted_var = f unless @predicted_var
-        end 
-      end
-    end
-    
-  end
-end
\ No newline at end of file
diff --git a/lib/utils/shims/opentox.rb b/lib/utils/shims/opentox.rb
deleted file mode 100644
index c10d535..0000000
--- a/lib/utils/shims/opentox.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-=begin
-* Name: opentox.rb
-* Description: Architecture shims
-* Author: Andreas Maunz <andreas@maunz.de>
-* Date: 10/2012
-=end
-
-# This avoids having to prefix everything with "RDF::" (e.g. "RDF::DC").
-# So that we can use our old code mostly as is.
-include RDF
-
-module OpenTox
-
-    # Help function to provide the metadata= functionality.
-    # Downward compatible to opentox-ruby.
-    # @param [Hash] Key-Value pairs with the metadata
-    # @return self
-    def metadata=(hsh) 
-      hsh.each {|k,v|
-        self[k]=v
-      }
-    end
-
-
-    ### Index Structures
-    
-    # Create parameter positions map
-    # @return [Hash] A hash with keys parameter names and values parameter positions
-    def build_parameter_positions
-      unless @parameter_positions
-        @parameters = parameters
-        @parameter_positions = @parameters.each_index.inject({}) { |h,idx|
-          h[@parameters[idx][DC.title.to_s]] = idx
-          h
-        }
-      end
-    end
-
-
-    ### Associative Search Operations
-    
-    # Search a model for a given parameter
-    # @param[String] The parameter title
-    # @return[Object] The parameter value
-    def find_parameter_value(title)
-      build_parameter_positions
-      res = @parameters[@parameter_positions[title]][OT.paramValue.to_s] if @parameter_positions[title]
-      res
-    end
-
-end
diff --git a/lib/utils/shims/task.rb b/lib/utils/shims/task.rb
deleted file mode 100644
index 7ac8a7d..0000000
--- a/lib/utils/shims/task.rb
+++ /dev/null
@@ -1,62 +0,0 @@
-=begin
-* Name: task.rb
-* Description: Task shims
-* Author: Andreas Maunz <andreas@maunz.de>
-* Date: 10/2012
-=end
-
-
-module OpenTox
-
-  # Shims for the Task class
-  class Task
-
-    def self.run(description, creator, subjectid=nil)
-       create($task[:uri],subjectid,{ RDF::DC.description => description, RDF::DC.creator => creator},&Proc.new) 
-    end
-
-    # Check status of a task
-    # @return [String] Status
-    def status
-      self[RDF::OT.hasStatus]
-    end
-    
-    def code
-      RestClientWrapper.head(@uri).code
-    end
-
-  end
-
-end
-
-
-module OpenTox
-
-  class SubTask
-    
-    def initialize(task, min, max)
-      #TODO add subtask code
-    end
-
-    def self.create(task, min, max)
-      if task
-        SubTask.new(task, min, max)
-      else
-        nil
-      end
-    end
-    
-    def waiting_for(task_uri)
-      #TODO add subtask code
-    end
-    
-    def progress(pct)
-      #TODO add subtask code
-    end
-    
-    def running?()
-      #TODO add subtask code
-    end
-  end
-  
-end
\ No newline at end of file
diff --git a/lib/utils/sparql/dataset.rb b/lib/utils/sparql/dataset.rb
deleted file mode 100644
index ecf55b6..0000000
--- a/lib/utils/sparql/dataset.rb
+++ /dev/null
@@ -1,75 +0,0 @@
-=begin
-* Name: dataset.rb
-* Description: Dataset SPARQL tools
-* Author: Andreas Maunz <andreas@maunz.de>
-* Date: 10/2012
-=end
-
-module OpenTox
-  class Dataset
-
-    # Load features via SPARQL (fast)
-    # @param [String] uri Dataset URI
-    # @return [Array] features OpenTox::Features in order
-    def self.find_features_sparql(uri)
-      sparql = "SELECT DISTINCT ?s FROM <#{uri}> WHERE {
-        ?s <#{RDF.type}> <#{RDF::OT.Feature}> ;
-           <#{RDF::OLO.index}> ?fidx
-        } ORDER BY ?fidx"
-      OpenTox::Backend::FourStore.query(sparql, "text/uri-list").split("\n").collect { |uri| OpenTox::Feature.new uri.strip }
-    end
-
-    # Load properties via SPARQL (fast)
-    # @param [Array] uris URIs (assumed ordered)
-    # @param [Hash] properties Properties (keys: user-defined identifier, values: rdf identifier as strings)
-    # @return [Array] types Properties in order of URIs
-    def self.find_props_sparql(uris, props)
-      selects = props.keys
-      conditions = selects.collect{ |k|
-        "<#{props[k]}> ?#{k.to_s}"
-      }
-      h={}
-      uris.each{ |uri|
-        sparql = "SELECT ?id #{selects.collect{|k| "?#{k.to_s}"}.join(" ")} FROM <#{uri}> WHERE { ?id #{conditions.join(";")} }"
-        res = OpenTox::Backend::FourStore.query(sparql, "text/uri-list")
-        res.split("\n").inject(h){ |h,row| 
-          values = row.split("\t")
-          id=values.shift
-          h[id] = {}
-          values.each_with_index { |val,idx|
-            h[id][selects[idx]] = [] unless h[id][selects[idx]]
-            h[id][selects[idx]] << val.to_s
-          }
-          h
-        }
-      }
-      h
-    end
-
-    # Load compounds via SPARQL (fast)
-    # @param [String] uri Dataset URI
-    # @return [Array] compounds Compounds in order
-    def self.find_compounds_sparql(uri)
-      sparql = "SELECT DISTINCT ?compound FROM <#{uri}> WHERE {
-        ?compound <#{RDF.type}> <#{RDF::OT.Compound}> ;
-                  <#{RDF::OLO.index}> ?cidx;
-        } ORDER BY ?cidx"
-      OpenTox::Backend::FourStore.query(sparql, "text/uri-list").split("\n").collect { |uri| OpenTox::Compound.new uri.strip }
-    end
-
-    # Load data entries via SPARQL (fast)
-    # @param [String] uri Dataset uri
-    # @return [Array] entries Data entries, ordered primarily over cols and secondarily over rows
-    def self.find_data_entries_sparql(uri)
-      sparql = "SELECT ?value FROM <#{uri}> WHERE {
-        ?data_entry <#{RDF::OLO.index}> ?cidx ;
-                    <#{RDF::OT.values}> ?v .
-        ?v          <#{RDF::OT.feature}> ?f;
-                    <#{RDF::OT.value}> ?value .
-        ?f          <#{RDF::OLO.index}> ?fidx.
-        } ORDER BY ?fidx ?cidx"
-      OpenTox::Backend::FourStore.query(sparql,"text/uri-list").split("\n").collect { |val| val.strip }
-    end
-
-  end
-end