1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
|
require 'csv'
require 'tempfile'
module OpenTox
class LazarPrediction < Dataset
field :creator, type: String
end
class FminerDataset < Dataset
field :training_algorithm, type: String
field :training_dataset_id, type: BSON::ObjectId
field :training_feature_id, type: BSON::ObjectId
field :training_parameters, type: Hash
end
class Dataset
include Mongoid::Document
attr_accessor :bulk
#attr_writer :data_entries
# associations like has_many, belongs_to deteriorate performance
field :feature_ids, type: Array, default: []
field :compound_ids, type: Array, default: []
field :data_entries_id, type: BSON::ObjectId
field :source, type: String
field :warnings, type: Array, default: []
def initialize params=nil
super params
@bulk = []
end
def save_all
dump = Marshal.dump(@data_entries)
file = Mongo::Grid::File.new(dump, :filename => "#{self.id.to_s}.data_entries")
data_entries_id = $gridfs.insert_one(file)
update(:data_entries_id => data_entries_id)
save
end
# Readers
def compounds
@compounds ||= self.compound_ids.collect{|id| OpenTox::Compound.find id}
@compounds
end
def features
@features ||= self.feature_ids.collect{|id| OpenTox::Feature.find(id)}
@features
end
def fill_nil_with n
(0 .. compound_ids.size-1).each do |i|
@data_entries[i] ||= []
(0 .. feature_ids.size-1).each do |j|
@data_entries[i][j] ||= n
end
end
end
def [](row,col)
#bad_request_error "Incorrect parameter type. The first argument is a OpenTox::Compound the second a OpenTox::Feature." unless compound.is_a? Compound and feature.is_a? Feature
#DataEntry.where(dataset_id: self.id, compound_id: compound.id, feature_id: feature.id).distinct(:value).first
#data_entries[compound_ids.index(compound.id)][feature_ids.index(feature.id)]
@data_entries[row,col]
end
def []=(row,col,v)
@data_entries ||= []
@data_entries[row] ||= []
#@data_entries ||= Array.new(compound_ids.size){Array.new(feature_ids.size)}
@data_entries[row][col] = v
end
def fingerprint(compound)
data_entries[compound_ids.index(compound.id)]
end
def data_entries
unless @data_entries
t = Time.now
@data_entries = Marshal.load($gridfs.find_one(_id: data_entries_id).data)
bad_request_error "Data entries (#{data_entries_id}) are not a 2D-Array" unless @data_entries.is_a? Array and @data_entries.first.is_a? Array
bad_request_error "Data entries (#{data_entries_id}) have #{@data_entries.size} rows, but dataset (#{id}) has #{compound_ids.size} compounds" unless @data_entries.size == compound_ids.size
bad_request_error "Data entries (#{data_entries_id}) have #{@data_entries..first.size} columns, but dataset (#{id}) has #{feature_ids.size} features" unless @data_entries.first.size == feature_ids.size
$logger.debug "Retrieving data: #{Time.now-t}"
end
@data_entries
end
# Find data entry values for a given compound and feature
# @param compound [OpenTox::Compound] OpenTox Compound object
# @param feature [OpenTox::Feature] OpenTox Feature object
# @return [Array] Data entry values
def values(compound, feature)
#data_entries.where(:compound_id => compound.id, :feature_id => feature.id).distinct(:value)
rows = (0 ... compound_ids.length).select { |r| compound_ids[r] == compound.id }
col = feature_ids.index feature.id
rows.collect{|row| data_entries[row][col]}
end
# Writers
def compounds=(compounds)
self.compound_ids = compounds.collect{|c| c.id}
end
def add_compound compound
self.compound_ids << compound.id
end
def features=(features)
self.feature_ids = features.collect{|f| f.id}
end
def add_feature feature
self.feature_ids << feature.id
end
def self.create compounds, features, warnings=[], source=nil
dataset = Dataset.new(:warnings => warnings)
dataset.compounds = compounds
dataset.features = features
dataset
end
# for prediction result datasets
# assumes that there are feature_ids with title prediction and confidence
# @return [Array] of Hashes with keys { :compound, :value ,:confidence } (compound value is object not uri)
# TODO
#def predictions
#end
# Serialisation
# converts dataset to csv format including compound smiles as first column, other column headers are feature titles
# @return [String]
def to_csv(inchi=false)
CSV.generate() do |csv| #{:force_quotes=>true}
csv << [inchi ? "InChI" : "SMILES"] + features.collect{|f| f.title}
compounds.each_with_index do |c,i|
csv << [inchi ? c.inchi : c.smiles] + data_entries[i]
end
end
end
# Methods for for validation service
# create a new dataset with the specified compounds and features
# @param compound_indices [Array] compound indices (integers)
# @param feats [Array] features objects
# @param metadata [Hash]
# @return [OpenTox::Dataset]
# TODO
def split( compound_indices, feats, metadata)
bad_request_error "Dataset.split : Please give compounds as indices" if compound_indices.size==0 or !compound_indices[0].is_a?(Fixnum)
bad_request_error "Dataset.split : Please give features as feature objects (given: #{feats})" if feats!=nil and feats.size>0 and !feats[0].is_a?(OpenTox::Feature)
dataset = OpenTox::Dataset.new
dataset.metadata = metadata
dataset.features = (feats ? feats : self.features)
compound_indices.each do |c_idx|
d = [ self.compounds[c_idx] ]
dataset.features.each_with_index.each do |f,f_idx|
d << (self.data_entries[c_idx] ? self.data_entries[c_idx][f_idx] : nil)
end
dataset << d
end
dataset.put
dataset
end
# maps a compound-index from another dataset to a compound-index from this dataset
# mapping works as follows:
# (compound c is the compound identified by the compound-index of the other dataset)
# * c occurs only once in this dataset? map compound-index of other dataset to index in this dataset
# * c occurs >1 in this dataset?
# ** number of occurences is equal in both datasets? assume order is preserved(!) and map accordingly
# ** number of occurences is not equal in both datasets? cannot map, raise error
# @param dataset [OpenTox::Dataset] dataset that should be mapped to this dataset (fully loaded)
# @param compound_index [Fixnum], corresponding to dataset
# TODO
def compound_index( dataset, compound_index )
compound_inchi = dataset.compounds[compound_index].inchi
self_indices = compound_indices(compound_inchi)
if self_indices==nil
nil
else
dataset_indices = dataset.compound_indices(compound_inchi)
if self_indices.size==1
self_indices.first
elsif self_indices.size==dataset_indices.size
# we do assume that the order is preseverd (i.e., the nth occurences in both datasets are mapped to each other)!
self_indices[dataset_indices.index(compound_index)]
else
raise "cannot map compound #{compound_inchi} from dataset #{dataset.id} to dataset #{self.id}, "+
"compound occurs #{dataset_indices.size} times and #{self_indices.size} times"
end
end
end
# returns the inidices of the compound in the dataset
# @param compound_inchi [String]
# @return [Array] compound index (position) of the compound in the dataset, array-size is 1 unless multiple occurences
# TODO
def compound_indices( compound_inchi )
unless defined?(@cmp_indices) and @cmp_indices.has_key?(compound_inchi)
@cmp_indices = {}
compounds().size.times do |i|
c = self.compounds[i].inchi
if @cmp_indices[c]==nil
@cmp_indices[c] = [i]
else
@cmp_indices[c] = @cmp_indices[c]+[i]
end
end
end
@cmp_indices[compound_inchi]
end
# Adding data methods
# (Alternatively, you can directly change @data["feature_ids"] and @data["compounds"])
# Create a dataset from file (csv,sdf,...)
# @param filename [String]
# @return [String] dataset uri
# TODO
#def self.from_sdf_file
#end
def bulk_write
time = Time.now
# Workaround for mongo bulk insertions (insertion of single data_entries is far too slow)
# Skip ruby JSON serialisation:
# - to_json is too slow to write to file
# - json (or bson) serialisation is probably causing very long parse times of Mongo::BulkWrite, or any other ruby insert operation
# this method causes a noticeable overhead compared to direct string serialisation (e.g. total processing time 16" instead of 12" for rat fminer dataset), but it can be reused at different places
dataset_id = self.id.to_s
f = Tempfile.new("#{dataset_id}.json","/tmp")
f.puts @bulk.collect{|row| "{'dataset_id': {'$oid': '#{dataset_id}'},'compound_id': {'$oid': '#{row[0]}'}, 'feature_id': {'$oid': '#{row[1]}'}, 'value': #{row[2]}}"}.join("\n")
f.close
$logger.debug "Write JSON file: #{Time.now-time}"
# TODO DB name from config
puts `mongoimport --db opentox --collection data_entries --type json --file #{f.path} 2>&1`
$logger.debug "Bulk import: #{Time.now-time}"
@bulk = []
end
def self.from_csv_file file, source=nil, bioassay=true
source ||= file
table = CSV.read file, :skip_blanks => true
dataset = Dataset.new(:source => source, :name => File.basename(file))
dataset.parse_table table, bioassay
dataset
end
# parse data in tabular format (e.g. from csv)
# does a lot of guesswork in order to determine feature types
def parse_table table, bioassay=true
time = Time.now
# features
feature_names = table.shift.collect{|f| f.strip}
warnings << "Duplicate features in table header." unless feature_names.size == feature_names.uniq.size
compound_format = feature_names.shift.strip
bad_request_error "#{compound_format} is not a supported compound format. Accepted formats: SMILES, InChI." unless compound_format =~ /SMILES|InChI/i
numeric = []
# guess feature types
feature_names.each_with_index do |f,i|
metadata = {}
values = table.collect{|row| val=row[i+1].to_s.strip; val.blank? ? nil : val }.uniq.compact
types = values.collect{|v| v.numeric? ? true : false}.uniq
if values.size == 0 # empty feature
elsif values.size > 5 and types.size == 1 and types.first == true # 5 max classes
metadata["numeric"] = true
numeric[i] = true
else
metadata["nominal"] = true
metadata["accept_values"] = values
numeric[i] = false
end
if bioassay
if metadata["numeric"]
feature = NumericBioAssay.find_or_create_by(metadata)
elsif metadata["nominal"]
feature = NominalBioAssay.find_or_create_by(metadata)
end
else
metadata.merge({:measured => false, :calculated => true})
if metadata["numeric"]
feature = NumericFeature.find_or_create_by(metadata)
elsif metadata["nominal"]
feature = NominalFeature.find_or_create_by(metadata)
end
end
feature_ids << OpenTox::Feature.find_or_create_by(metadata).id
end
#feature_ids = dataset.features.collect{|f| f.id.to_s}
$logger.debug "Feature values: #{Time.now-time}"
time = Time.now
r = -1
compound_time = 0
value_time = 0
# compounds and values
@data_entries = Array.new(table.size){Array.new(table.first.size-1)}
table.each_with_index do |vals,i|
ct = Time.now
identifier = vals.shift
begin
case compound_format
when /SMILES/i
compound = OpenTox::Compound.from_smiles(identifier)
if compound.inchi.empty?
warnings << "Cannot parse #{compound_format} compound '#{compound.strip}' at position #{i+2}, all entries are ignored."
next
end
when /InChI/i
# compounds and values
compound = OpenTox::Compound.from_inchi(identifier)
end
rescue
warnings << "Cannot parse #{compound_format} compound '#{compound}' at position #{i+2}, all entries are ignored."
next
end
compound_time += Time.now-ct
compound_ids << compound.id
r += 1
unless vals.size == feature_ids.size # way cheaper than accessing features
warnings << "Number of values at position #{i+2} is different than header size (#{vals.size} vs. #{features.size}), all entries are ignored."
next
end
cid = compound.id.to_s
vals.each_with_index do |v,j|
if v.blank?
warnings << "Empty value for compound '#{identifier}' (row #{r+2}) and feature '#{feature_names[j]}' (column #{j+2})."
next
elsif numeric[j]
@data_entries[i][j] = v.to_f
#dataset.bulk << [cid,feature_ids[j],v.to_f]
else
@data_entries[i][j] = v.strip
#dataset.bulk << [cid,feature_ids[j],v.strip]
end
end
end
compounds.duplicates.each do |compound|
positions = []
compounds.each_with_index{|c,i| positions << i+1 if !c.blank? and c.inchi == compound.inchi}
warnings << "Duplicate compound #{compound.inchi} at rows #{positions.join(', ')}. Entries are accepted, assuming that measurements come from independent experiments."
end
$logger.debug "Value parsing: #{Time.now-time} (Compound creation: #{compound_time})"
time = Time.now
#dataset.bulk_write
save_all
$logger.debug "Saving: #{Time.now-time}"
end
end
end
|