1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
|
module OpenTox
# Ruby wrapper for OpenTox Dataset Webservices (http://opentox.org/dev/apis/api-1.2/dataset).
#
# Examples:
# require "opentox-ruby-api-wrapper"
#
# # Creating datasets
#
# # create an empty dataset
# dataset = OpenTox::Dataset.new
# # create an empty dataset with URI
# # this does not load data from the dataset service - use one of the load_* methods
# dataset = OpenTox::Dataset.new("http:://webservices.in-silico/ch/dataset/1")
# # create new dataset and sav it to obtain a URI
# dataset = OpenTox::Dataset.create
# # create a new dataset from yaml representation
# dataset = OpenTox::Dataset.from_yaml
# # create a new dataset from CSV string
# csv_string = "SMILES, Toxicity\nc1ccccc1N, true"
# dataset = OpenTox::Dataset.from_csv(csv_string)
#
# # Loading data
# # Datasets created with OpenTox::Dataset.new(uri) are empty by default
# # Invoking one of the following functions will load data into the object
#
# # create an empty dataset with URI
# dataset = OpenTox::Dataset.new("http:://webservices.in-silico/ch/dataset/1")
# # loads (and returns) only metadata
# dataset.load_metadata
# # loads (and returns) only compounds
# dataset.load_compounds
# # loads (and returns) only features
# dataset.load_features
# # load all data from URI
# dataset.load_all
#
# # Getting dataset representations
#
# dataset = OpenTox::Dataset.new("http:://webservices.in-silico/ch/dataset/1")
# dataset.load_all
# # OWL-DL (RDF/XML)
# dataset.rdfxml
# # OWL-DL (Ntriples)
# dataset.ntriples
# # YAML
# dataset.yaml
# # CSV
# dataset.csv
#
# # Modifying datasets
#
# # insert a statement (compound_uri,feature_uri,value)
# dataset.add "http://webservices.in-silico.ch/compound/InChI=1S/C6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9", "http://webservices.in-silico.ch/dataset/1/feature/hamster_carcinogenicity", true
#
#
# # Saving datasets
# # save dataset at dataset service
# dataset.save
#
# # Deleting datasets
# # delete dataset (also at dataset service)
# dataset.delete
class Dataset
include OtObject
attr_reader :features, :compounds, :data_entries, :metadata
attr_writer :metadata
# Create dataset with optional URI
def initialize(uri=nil)
super uri
@features = {}
@compounds = []
@data_entries = {}
end
# Create and save an empty dataset (assigns URI to dataset)
def self.create(uri=CONFIG[:services]["opentox-dataset"])
dataset = Dataset.new
dataset.uri = RestClientWrapper.post(uri,{}).to_s.chomp
dataset
end
# Get all datasets from a service
# def self.all(uri=CONFIG[:services]["opentox-dataset"])
# RestClientWrapper.get(uri,:accept => "text/uri-list").to_s.each_line.collect{|u| Dataset.new(u)}
# end
# Create a dataset from YAML string
def self.from_yaml(yaml)
dataset = Dataset.create
dataset.copy YAML.load(yaml)
dataset
end
# Create dataset from CSV string (format specification: http://toxcreate.org/help)
# - loads data_entries, compounds, features
# - sets metadata (warnings) for parser errors
# - you will have to set remaining metadata manually
def self.from_csv(csv)
dataset = Dataset.create
Parser::Spreadsheet.new(dataset).load_csv(csv)
dataset
end
# Create dataset from Spreadsheet book (created with roo gem http://roo.rubyforge.org/, excel format specification: http://toxcreate.org/help))
# - loads data_entries, compounds, features
# - sets metadata (warnings) for parser errors
# - you will have to set remaining metadata manually
def self.from_spreadsheet(book)
dataset = Dataset.create
Parser::Spreadsheet.new(dataset).load_excel(book)
dataset
end
# Load and return metadata of a Dataset object
def load_metadata
#if (CONFIG[:yaml_hosts].include?(URI.parse(@uri).host))
#add_metadata YAML.load(RestClientWrapper.get(File.join(@uri,"metadata"), :accept => "application/x-yaml"))
#else
add_metadata Parser::Owl::Dataset.new(@uri).metadata
#end
self.uri = @uri if @uri # keep uri
@metadata
end
# Load all data (metadata, data_entries, compounds and features) from URI
def load_all
if (CONFIG[:yaml_hosts].include?(URI.parse(@uri).host))
copy YAML.load(RestClientWrapper.get(@uri, :accept => "application/x-yaml"))
else
parser = Parser::Owl::Dataset.new(@uri)
copy parser.load_uri
end
end
# Load and return all compound URIs
def load_compounds
RestClientWrapper.get(File.join(uri,"compounds"),:accept=> "text/uri-list").to_s.each_line do |compound_uri|
@compounds << compound_uri.chomp
end
@compounds.uniq!
end
# Load all feature URIs
def load_features
RestClientWrapper.get(File.join(uri,"features"),:accept=> "text/uri-list").to_s.each_line do |feature_uri|
@features[feature_uri.chomp] = Feature.new(feature_uri.chomp).load_metadata
end
@features
end
# Get YAML representation
def yaml
self.to_yaml
end
# Get Excel representation, returns a Spreadsheet::Workbook which can be written with the 'spreadsheet' gem (data_entries only, metadata will )
def excel
Serializer::Spreadsheets.new(self).excel
end
# Get CSV string representation (data_entries only, metadata will be discarded)
def csv
Serializer::Spreadsheets.new(self).csv
end
# Get OWL-DL in ntriples format
def ntriples
s = Serializer::Owl.new
s.add_dataset(self)
s.ntriples
end
# Get OWL-DL in RDF/XML format
def rdfxml
s = Serializer::Owl.new
s.add_dataset(self)
s.rdfxml
end
# Insert a statement (compound_uri,feature_uri,value)
def add (compound,feature,value)
@compounds << compound unless @compounds.include? compound
@features[feature] = {} unless @features[feature]
@data_entries[compound] = {} unless @data_entries[compound]
@data_entries[compound][feature] = [] unless @data_entries[compound][feature]
@data_entries[compound][feature] << value
end
# Add metadata (hash with predicate_uri => value)
def add_metadata(metadata)
metadata.each { |k,v| @metadata[k] = v }
end
# Copy a dataset (rewrites URI)
def copy(dataset)
@metadata = dataset.metadata
@data_entries = dataset.data_entries
@compounds = dataset.compounds
@features = dataset.features
if @uri
self.uri = @uri
else
@uri = dataset.metadata[XSD.anyUri]
end
end
# save dataset (overwrites existing dataset)
def save
# TODO: rewrite feature URI's ??
# create dataset if uri empty
@compounds.uniq!
RestClientWrapper.post(@uri,{:content_type => "application/x-yaml"},self.to_yaml)
end
# Delete dataset at the dataset service
def delete
RestClientWrapper.delete @uri
end
end
end
#########################################################
# kept for backward compatibility, may have to be fixed #
#########################################################
=begin
def from_owl(owl)
# creates dataset object from Opentox::Owl object
# use Dataset.find( <uri> ) to load dataset from rdf-supporting datasetservice
# note: does not load all feature values, as this is time consuming
raise "invalid param" unless owl.is_a?(OpenTox::Owl)
@metadata[DC.title] = owl.get("title")
@metadata[DC.creator] = owl.get("creator")
@metadata[XSD.anyUri] = owl.uri
# when loading a dataset from owl, only compound- and feature-uris are loaded
owl.load_dataset(@compounds, @features)
# all features are marked as dirty
# as soon as a feature-value is requested all values for this feature are loaded from the rdf
@dirty_features = @features.dclone
@owl = owl
end
def self.find(uri, accept_header=nil)
unless accept_header
if (CONFIG[:yaml_hosts].include?(URI.parse(uri).host))
accept_header = 'application/x-yaml'
else
accept_header = "application/rdf+xml"
end
end
case accept_header
when "application/x-yaml"
LOGGER.debug "DATASET: "+ uri
LOGGER.debug RestClientWrapper.get(uri.to_s.strip, :accept => 'application/x-yaml').to_s
d = YAML.load RestClientWrapper.get(uri.to_s.strip, :accept => 'application/x-yaml').to_s
#d.uri = @metadata[XSD.anyUri] unless d.uri
when "application/rdf+xml"
owl = OpenTox::Owl.from_uri(uri.to_s.strip, "Dataset")
d = Dataset.new(owl)
else
raise "cannot get datset with accept header: "+accept_header.to_s
end
d
end
# converts a dataset represented in owl to yaml
# (uses a temporary dataset)
# note: to_yaml is overwritten, loads complete owl dataset values
def self.owl_to_yaml( owl_data, uri)
owl = OpenTox::Owl.from_data(owl_data, uri, "Dataset")
d = Dataset.new(owl)
d.to_yaml
end
# creates a new dataset, using only those compounsd specified in new_compounds
# returns uri of new dataset
def create_new_dataset( new_compounds, new_features, new_title, new_creator )
LOGGER.debug "create new dataset with "+new_compounds.size.to_s+"/"+compounds.size.to_s+" compounds"
raise "no new compounds selected" unless new_compounds and new_compounds.size>0
# load require features
if ((defined? @dirty_features) && (@dirty_features & new_features).size > 0)
(@dirty_features & new_features).each{|f| load_feature_values(f)}
end
dataset = OpenTox::Dataset.new
dataset.title = new_title
dataset.creator = new_creator
dataset.features = new_features
dataset.compounds = new_compounds
# Copy dataset data for compounds and features
# PENDING: why storing feature values in an array?
new_compounds.each do |c|
data_c = []
raise "no data for compound '"+c.to_s+"'" if @data[c]==nil
@data[c].each do |d|
m = {}
new_features.each do |f|
m[f] = d[f]
end
data_c << m
end
dataset.data[c] = data_c
end
return dataset.save
end
# returns classification value
def get_predicted_class(compound, feature)
v = get_value(compound, feature)
if v.is_a?(Hash)
k = v.keys.grep(/classification/).first
unless k.empty?
#if v.has_key?(:classification)
return v[k]
else
return "no classification key"
end
elsif v.is_a?(Array)
raise "predicted class value is an array\n"+
"value "+v.to_s+"\n"+
"value-class "+v.class.to_s+"\n"+
"dataset "+self.uri.to_s+"\n"+
"compound "+compound.to_s+"\n"+
"feature "+feature.to_s+"\n"
else
return v
end
end
# returns regression value
def get_predicted_regression(compound, feature)
v = get_value(compound, feature)
if v.is_a?(Hash)
k = v.keys.grep(/regression/).first
unless k.empty?
return v[k]
else
return "no regression key"
end
elsif v.is_a?(Array)
raise "predicted regression value is an array\n"+
"value "+v.to_s+"\n"+
"value-class "+v.class.to_s+"\n"+
"dataset "+self.uri.to_s+"\n"+
"compound "+compound.to_s+"\n"+
"feature "+feature.to_s+"\n"
else
return v
end
end
# returns prediction confidence if available
def get_prediction_confidence(compound, feature)
v = get_value(compound, feature)
if v.is_a?(Hash)
k = v.keys.grep(/confidence/).first
unless k.empty?
#if v.has_key?(:confidence)
return v[k].abs
#return v["http://ot-dev.in-silico.ch/model/lazar#confidence"].abs
else
# PENDING: return nil isntead of raising an exception
raise "no confidence key"
end
else
LOGGER.warn "no confidence for compound: "+compound.to_s+", feature: "+feature.to_s
return 1
end
end
# return compound-feature value
def get_value(compound, feature)
if (defined? @dirty_features) && @dirty_features.include?(feature)
load_feature_values(feature)
end
v = @data[compound]
return nil if v == nil # missing values for all features
if v.is_a?(Array)
# PENDING: why using an array here?
v.each do |e|
if e.is_a?(Hash)
if e.has_key?(feature)
return e[feature]
end
else
raise "invalid internal value type"
end
end
return nil #missing value
else
raise "value is not an array\n"+
"value "+v.to_s+"\n"+
"value-class "+v.class.to_s+"\n"+
"dataset "+self.uri.to_s+"\n"+
"compound "+compound.to_s+"\n"+
"feature "+feature.to_s+"\n"
end
end
# loads specified feature and removes dirty-flag, loads all features if feature is nil
def load_feature_values(feature=nil)
if feature
raise "feature already loaded" unless @dirty_features.include?(feature)
@owl.load_dataset_feature_values(@compounds, @data, [feature])
@dirty_features.delete(feature)
else
@data = {} unless @data
@owl.load_dataset_feature_values(@compounds, @data, @dirty_features)
@dirty_features.clear
end
end
# overwrite to yaml:
# in case dataset is loaded from owl:
# * load all values
def to_yaml
# loads all features
if ((defined? @dirty_features) && @dirty_features.size > 0)
load_feature_values
end
super
end
# * remove @owl from yaml, not necessary
def to_yaml_properties
super - ["@owl"]
end
end
end
=end
|