summaryrefslogtreecommitdiff
path: root/lib/dataset.rb
blob: 2a777bb8801c1d9b7f26d971eb0604d650227e6e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
require 'csv'

module OpenTox

  # Ruby wrapper for OpenTox Dataset Webservices (http://opentox.org/dev/apis/api-1.2/dataset).
  class Dataset
    include Mongoid::Document

    field :feature_ids, type: Array, default: []
    field :inchis, type: Array, default: []
    field :data_entries, type: Array, default: []
    field :warnings, type: Array, default: []
    field :source, type: String

    # TODO subclass Datasets
    field :dependent_variables, type: BSON::ObjectId
    field :predicted_variables, type: Array

    # Readers

    def compounds
      inchis.collect{|i| OpenTox::Compound.new i}
    end

    def features
      self.feature_ids.collect{|id| OpenTox::Feature.find(id)}
    end

    # Writers

    def compounds=(compounds)
      self.inchis = compounds.collect{|c| c.inchi}
    end

    def add_compound(compound)
      self.inchis << compound.id
    end

    def features=(features)
      self.feature_ids = features.collect{|f| f.id}
    end

    def add_feature(feature)
      self.feature_ids << feature.id
    end

    # Find data entry values for a given compound and feature
    # @param compound [OpenTox::Compound] OpenTox Compound object
    # @param feature [OpenTox::Feature] OpenTox Feature object
    # @return [Array] Data entry values
    def values(compound, feature)
      rows = (0 ... inchis.length).select { |r| inchis[r] == compound.inchi }
      col = feature_ids.index feature.id
      rows.collect{|row| data_entries[row][col]}
    end

    # Convenience methods to search by compound/feature URIs

    # Search a dataset for a feature given its URI
    # @param uri [String] Feature URI
    # @return [OpenTox::Feature] Feature object, or nil if not present
    #def find_feature_uri(uri)
      #features.select{|f| f.uri == uri}.first
    #end

    # Search a dataset for a compound given its URI
    # @param uri [String] Compound URI
    # @return [OpenTox::Compound] Compound object, or nil if not present
    def find_compound_uri(uri)
      compounds.select{|f| f.uri == uri}.first
    end

    # for prediction result datasets
    # assumes that there are feature_ids with title prediction and confidence
    # @return [Array] of Hashes with keys { :compound, :value ,:confidence } (compound value is object not uri)
    def predictions
      predictions = []
      prediction_feature = nil
      confidence_feature = nil
      metadata[RDF::OT.predictedVariables].each do |uri|
        feature = OpenTox::Feature.new uri
        case feature.title
        when /prediction$/
          prediction_feature = feature
        when /confidence$/
          confidence_feature = feature
        end
      end
      if prediction_feature and confidence_feature
        compounds.each do |compound|
          value = values(compound,prediction_feature).first
          value = value.to_f if prediction_feature[RDF.type].include? RDF::OT.NumericFeature and ! prediction_feature[RDF.type].include? RDF::OT.StringFeature
          confidence = values(compound,confidence_feature).first.to_f
          predictions << {:compound => compound, :value => value, :confidence => confidence} if value and confidence
        end
      end
      predictions
    end

    # Adding data methods
    # (Alternatively, you can directly change @data["feature_ids"] and @data["compounds"])

    # Create a dataset from file (csv,sdf,...)
    # @param filename [String]
    # @return [String] dataset uri
    def upload filename, wait=true
      self.title = File.basename(filename)
      self.source = filename
      table = CSV.read filename, :skip_blanks => true
      from_table table
      save
    end

    # @param compound [OpenTox::Compound]
    # @param feature [OpenTox::Feature]
    # @param value [Object] (will be converted to String)
    # @return [Array] data_entries
    def add_data_entry compound, feature, value
      # TODO: optimize
      add_compound compound unless self.compounds.include?(compound)
      row = self.compounds.index(compound)
      add_feature feature unless self.features.include?(feature)
      col = self.features.index(feature)
      if self.data_entries[row] and self.data_entries[row][col] # duplicated values
        add_compound compound
        row = self.compounds.rindex(compound)
      end
      if value
        self.data_entries[row] ||= []
        self.data_entries[row][col] = value
      end
    end


    # TODO: remove? might be dangerous if feature ordering is incorrect
    # MG: I would not remove this because add_data_entry is very slow (4 times searching in arrays)
    # CH: do you have measurements? compound and feature arrays are not that big, I suspect that feature search/creation is the time critical step
    # @param row [Array]
    # @example
    #   d = Dataset.new
    #   d.features << Feature.new(a)
    #   d.features << Feature.new(b)
    #   d << [ Compound.new("c1ccccc1"), feature-value-a, feature-value-b ]
    def << row
      compound = row.shift # removes the compound from the array
      bad_request_error "Dataset features are empty." unless feature_ids
      bad_request_error "Row size '#{row.size}' does not match features size '#{feature_ids.size}'." unless row.size == feature_ids.size
      bad_request_error "First column is not a OpenTox::Compound" unless compound.class == OpenTox::Compound
      self.inchis << compound.inchi
      self.data_entries << row
    end

    # Serialisation
    
    # converts dataset to csv format including compound smiles as first column, other column headers are feature titles
    # @return [String]
    def to_csv(inchi=false)
      CSV.generate() do |csv| #{:force_quotes=>true}
        csv << [inchi ? "InChI" : "SMILES"] + features.collect{|f| f.title}
        compounds.each_with_index do |c,i|
          csv << [inchi ? c.inchi : c.smiles] + data_entries[i]
        end
      end
    end

    # Methods for for validation service

    # create a new dataset with the specified compounds and features
    # @param compound_indices [Array] compound indices (integers)
    # @param feats [Array] features objects
    # @param metadata [Hash]
    # @return [OpenTox::Dataset]
    def split( compound_indices, feats, metadata)

      bad_request_error "Dataset.split : Please give compounds as indices" if compound_indices.size==0 or !compound_indices[0].is_a?(Fixnum)
      bad_request_error "Dataset.split : Please give features as feature objects (given: #{feats})" if feats!=nil and feats.size>0 and !feats[0].is_a?(OpenTox::Feature)
      dataset = OpenTox::Dataset.new
      dataset.metadata = metadata
      dataset.features = (feats ? feats : self.features)
      compound_indices.each do |c_idx|
        d = [ self.compounds[c_idx] ]
        dataset.features.each_with_index.each do |f,f_idx|
          d << (self.data_entries[c_idx] ? self.data_entries[c_idx][f_idx] : nil)
        end
        dataset << d
      end
      dataset.put
      dataset
    end


    # maps a compound-index from another dataset to a compound-index from this dataset
    # mapping works as follows:
    # (compound c is the compound identified by the compound-index of the other dataset)
    # * c occurs only once in this dataset? map compound-index of other dataset to index in this dataset
    # * c occurs >1 in this dataset?
    # ** number of occurences is equal in both datasets? assume order is preserved(!) and map accordingly
    # ** number of occurences is not equal in both datasets? cannot map, raise error
    # @param dataset [OpenTox::Dataset] dataset that should be mapped to this dataset (fully loaded)
    # @param compound_index [Fixnum], corresponding to dataset
    def compound_index( dataset, compound_index )
      compound_inchi = dataset.compounds[compound_index].inchi
      self_indices = compound_indices(compound_inchi)
      if self_indices==nil
        nil
      else
        dataset_indices = dataset.compound_indices(compound_inchi)
        if self_indices.size==1
          self_indices.first
        elsif self_indices.size==dataset_indices.size
          # we do assume that the order is preseverd (i.e., the nth occurences in both datasets are mapped to each other)!
          self_indices[dataset_indices.index(compound_index)]
        else
          raise "cannot map compound #{compound_inchi} from dataset #{dataset.id} to dataset #{self.id}, "+
            "compound occurs #{dataset_indices.size} times and #{self_indices.size} times"
        end
      end
    end

    # returns the inidices of the compound in the dataset
    # @param compound_inchi [String]
    # @return [Array] compound index (position) of the compound in the dataset, array-size is 1 unless multiple occurences
    def compound_indices( compound_inchi )
      unless defined?(@cmp_indices) and @cmp_indices.has_key?(compound_inchi)
        @cmp_indices = {}
        compounds().size.times do |i|
          c = self.compounds[i].inchi
          if @cmp_indices[c]==nil
            @cmp_indices[c] = [i]
          else
            @cmp_indices[c] = @cmp_indices[c]+[i]
          end
        end
      end
      @cmp_indices[compound_inchi]
    end

    # returns compound feature value using the compound-index and the feature_uri
    def data_entry_value(compound_index, feature_uri)
      data_entries(true) if @data["data_entries"].empty?
      col = @data["features"].collect{|f| f.uri}.index feature_uri
      @data["data_entries"][compound_index] ?  @data["data_entries"][compound_index][col] : nil
    end

    def from_table table

      # features
      feature_names = table.shift.collect{|f| f.strip}
      self.warnings << "Duplicate features in table header." unless feature_names.size == feature_names.uniq.size
      compound_format = feature_names.shift.strip
      bad_request_error "#{compound_format} is not a supported compound format. Accepted formats: SMILES, InChI." unless compound_format =~ /SMILES|InChI/i
      ignored_feature_indices = []
      numeric = []
      feature_names.each_with_index do |f,i|
        values = table.collect{|row| val=row[i+1].to_s.strip; val.blank? ? nil : val }.uniq.compact
        types = values.collect{|v| v.numeric? ? true : false}.uniq
        metadata = {"title" => f}
        if values.size == 0 # empty feature
        elsif  values.size > 5 and types.size == 1 and types.first == true # 5 max classes
          metadata["numeric"] = true
          numeric[i] = true
        else
          metadata["nominal"] = true
          metadata["string"] = true
          metadata["accept_values"] = values
          numeric[i] = false
        end
        feature = OpenTox::Feature.find_or_create_by metadata
        self.feature_ids << feature.id unless feature.nil?
      end

      # compounds and values
      r = -1
      table.each_with_index do |values,j|
        compound = values.shift
        begin
          case compound_format
          when /SMILES/i
            c = OpenTox::Compound.from_smiles(compound)
            if c.inchi.empty?
              self.warnings << "Cannot parse #{compound_format} compound '#{compound.strip}' at position #{j+2}, all entries are ignored."
              next
            else
              inchi = c.inchi
            end
          when /InChI/i
            # TODO validate inchi
            inchi = compound
          else
            raise "wrong compound format" #should be checked above
          end
        rescue
          self.warnings << "Cannot parse #{compound_format} compound '#{compound}' at position #{j+2}, all entries are ignored."
          next
        end
        
        r += 1
        self.inchis << inchi
        unless values.size == self.feature_ids.size
          self.warnings << "Number of values at position #{j+2} (#{values.size}) is different than header size (#{self.feature_ids.size}), all entries are ignored."
          next
        end

        self.data_entries << []
        values.each_with_index do |v,i|
          if v.blank?
            self.data_entries.last << nil
            self.warnings << "Empty value for compound '#{compound}' (row #{r+2}) and feature '#{feature_names[i]}' (column #{i+2})."
            next
          elsif numeric[i]
            self.data_entries.last << v.to_f
          else
            self.data_entries.last << v.strip
          end
        end
      end
      self.inchis.duplicates.each do |inchi|
        positions = []
        self.inchis.each_with_index{|c,i| positions << i+1 if !c.blank? and c == inchi}
        self.warnings << "Duplicate compound #{inchi} at rows #{positions.join(', ')}. Entries are accepted, assuming that measurements come from independent experiments." 
      end
    end
  end
end