mongo batch import workaround

15 files changed, 321 insertions, 134 deletions

diff --git a/standalone.rb b/standalone.rb
new file mode 100644
index 0000000..907fa2c
--- /dev/null
+++ b/standalone.rb
@@ -0,0 +1,26 @@
+require 'minitest/autorun'
+[
+  "algorithm",
+  "compound",
+  "dataset-long",
+  "dataset",
+  "descriptor-long",
+  "descriptor",
+  "edit_objects",
+  #"error",
+  "fminer",
+  "lazar-fminer",
+  "lazar-long",
+#"lazar-models",
+  #"lazar-physchem-long",
+  #"lazar-physchem-short",
+#"lazarweb",
+  #"task",
+  #"validation-long",
+  #"validation-short",
+  #"validation_util",*
+].each {|t| require_relative File.join("test", t+".rb")}
+
+#require './test/store_query.rb'
+#require './test/authorization.rb'
+#require './test/policy.rb'
diff --git a/test/compound.rb b/test/compound.rb
index 17841d1..70271fc 100644
--- a/test/compound.rb
+++ b/test/compound.rb
@@ -31,15 +31,24 @@ class CompoundTest < MiniTest::Test
     assert_equal "c1ccccc1", c.smiles
   end
 
+  def test_sdf_import
+    c = OpenTox::Compound.from_sdf File.read(File.join DATA_DIR, "acetaldehyde.sdf")
+    assert_equal "InChI=1S/C2H4O/c1-2-3/h2H,1H3", c.inchi
+    assert_equal "CC=O", c.smiles
+    assert c.names.include? "Acetylaldehyde"
+  end
+
+  def test_sdf_export
+    c = OpenTox::Compound.from_smiles "CC=O"
+    assert_match /7  6  0  0  0  0  0  0  0  0999 V2000/, c.sdf
+  end
+
   def test_compound_image
     c = OpenTox::Compound.from_inchi "InChI=1S/C6H6/c1-2-4-6-5-3-1/h1-6H"
     testbild = "/tmp/testbild.png"
-    f = File.open(testbild, "w")
-    f.puts c.png
-    f.close
+    File.open(testbild, "w"){|f| f.puts c.png}
     assert_match "image/png", `file -b --mime-type /tmp/testbild.png`
     File.unlink(testbild)
-    #assert_match /^\x89PNG/, c.png #32bit only?
   end
 
   # OpenBabel segfaults randomly during inchikey calculation
@@ -47,8 +56,6 @@ class CompoundTest < MiniTest::Test
     c = OpenTox::Compound.from_inchi "InChI=1S/C6H6/c1-2-4-6-5-3-1/h1-6H"
     assert_equal "UHOVQNZJYSORNB-UHFFFAOYSA-N", c.inchikey
   end
-=begin
-=end
 
   def test_cid
     c = OpenTox::Compound.from_inchi "InChI=1S/C6H6/c1-2-4-6-5-3-1/h1-6H"
diff --git a/test/data/acetaldehyde.sdf b/test/data/acetaldehyde.sdf
new file mode 100644
index 0000000..de8dbd1
--- /dev/null
+++ b/test/data/acetaldehyde.sdf
@@ -0,0 +1,14 @@
+

+

+

+  3  2  0  0  0  0  0  0  0  0  1 V2000

+    2.3030   -0.6656    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0

+    1.1515    0.0000    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0

+    0.0000   -0.6656    0.0000 O   0  0  0  0  0  0  0  0  0  0  0  0

+  1  2  1  0  0  0  0

+  2  3  2  0  0  0  0

+M  END

+>  <ActivityOutcome_CPDBAS_Hamster>

+active

+

+$$$$

diff --git a/test/data_entry.rb b/test/data_entry.rb
new file mode 100644
index 0000000..13a2884
--- /dev/null
+++ b/test/data_entry.rb
@@ -0,0 +1,92 @@
+require_relative "setup.rb"
+
+class DataEntryTest < MiniTest::Test
+
+  def test_create
+
+    # add features
+    features = ["test1", "test2"].collect do |title|
+      NumericBioAssay.find_or_create_by( :title => title, :numeric => true)
+    end
+
+    compounds = []
+    input = [
+      ["c1ccccc1NN",1,2],
+      ["CC(C)N",4,5],
+      ["C1C(C)CCCC1",6,7],
+    ]
+    input.each do |row|
+      smi = row.shift
+      compound = Compound.find_or_create_by(:smiles => smi)
+      compounds << compound
+      row.each_with_index do |value,i|
+        DataEntry.find_or_create compound, features[i], value
+      end
+    end
+    
+    assert_equal 3, compounds.size
+    assert_equal 2, features.size
+    input.each_with_index do |row,i|
+      row.each_with_index do |v,j|
+        assert_equal DataEntry[compounds[i],features[j]], input[i][j]
+      end
+    end
+  end
+
+  def test_create_from_file
+    d = OpenTox::Dataset.from_csv_file File.join(DATA_DIR,"EPAFHM.mini.csv")
+    assert_equal OpenTox::Dataset, d.class
+    refute_nil d.warnings
+    assert_match /row 13/, d.warnings.join
+    assert_match "EPAFHM.mini.csv",  d.source
+    assert_equal 1, d.features.size
+    feature = d.features.first
+    assert_kind_of NumericBioAssay, feature
+    assert_match "EPAFHM.mini.csv",  feature.source
+    assert_equal 0.0113, DataEntry[d.compounds.first, feature]
+    assert_equal 0.00323, DataEntry[d.compounds[5], feature]
+  end
+
+  def test_upload_kazius
+    d = OpenTox::Dataset.from_csv_file File.join DATA_DIR, "kazius.csv"
+    assert_empty d.warnings
+    #  493 COC1=C(C=C(C(=C1)Cl)OC)Cl,1
+    c = d.compounds[491]
+    assert_equal c.smiles, "COc1cc(c(cc1Cl)OC)Cl"
+    assert_equal DataEntry[c,d.features.first], 1
+  end
+
+  def test_upload_feature_dataset
+    t1 = Time.now
+    f = File.join DATA_DIR, "rat_feature_dataset.csv"
+    d = OpenTox::Dataset.from_csv_file f
+    assert_equal 458, d.features.size
+    d.save
+    t2 = Time.now
+    p "Upload: #{t2-t1}"
+    d2 = OpenTox::Dataset.find d.id
+    t3 = Time.now
+    p "Dowload: #{t3-t2}"
+    assert_equal d.features.size, d2.features.size
+    csv = CSV.read f
+    assert_equal csv.size-1, d2.compounds.size
+    assert_equal csv.first.size-1, d2.features.size
+    # asserting complete ds
+    3.times do
+      cid = rand(d.compounds.size)
+      3.times do
+        fid = rand(d.features.size)
+        # TODO data access is slow
+        assert_equal csv[cid+1][fid+1].to_i, DataEntry[d2.compounds[cid],d2.features[fid]]
+      end
+    end
+    #assert_equal csv.size-1, d.data_entries.size
+    d2.delete
+    assert_raises Mongoid::Errors::DocumentNotFound do
+      Dataset.find d.id
+    end
+  end
+
+
+end
+
diff --git a/test/descriptor.rb b/test/descriptor.rb
index 89ea457..7ed0abd 100644
--- a/test/descriptor.rb
+++ b/test/descriptor.rb
@@ -1,39 +1,40 @@
 require_relative "setup.rb"
 
-begin
-  puts "Service URI is: #{$algorithm[:uri]}"
-rescue
-  puts "Configuration Error: $algorithm[:uri] is not defined in: " + File.join(ENV["HOME"],".opentox","config","test.rb")
-  exit
-end
-
 class DescriptorTest < MiniTest::Test
 
   def test_list
-    skip "TODO: Test descriptor list"
-  end
-
-  def test_lookup
-    skip "TODO: Test descriptor lookup"
+    # check available descriptors
+    @descriptors = OpenTox::Algorithm::Descriptor::DESCRIPTORS.keys
+    assert_equal 111,@descriptors.size,"wrong num physchem descriptors"
+    @descriptor_values = OpenTox::Algorithm::Descriptor::DESCRIPTOR_VALUES
+    assert_equal 356,@descriptor_values.size,"wrong num physchem descriptors"
+    sum = 0
+    [ @descriptors, @descriptor_values ].each do |desc|
+      {"Openbabel"=>16,"Cdk"=>(desc==@descriptors ? 50 : 295),"Joelib"=>45}.each do |k,v|
+        assert_equal v,desc.select{|x| x=~/^#{k}\./}.size,"wrong num #{k} descriptors"
+        sum += v
+      end
+    end
+    assert_equal (111+356),sum
   end
 
   def test_smarts
     c = OpenTox::Compound.from_smiles "N=C=C1CCC(=F=FO)C1"
     result = OpenTox::Algorithm::Descriptor.smarts_match c, "FF"
-    assert_equal 1, result[c]["FF"]
-    smarts = {"CC"=>1, "C"=>1, "C=C"=>1, "CO"=>0, "FF"=>1, "C1CCCC1"=>1, "NN"=>0}
-    result = OpenTox::Algorithm::Descriptor.smarts_match c, smarts.keys
-    assert_equal smarts, result[c]
-    smarts_count = {"CC"=>10, "C"=>6, "C=C"=>2, "CO"=>0, "FF"=>2, "C1CCCC1"=>10, "NN"=>0}
-    result = OpenTox::Algorithm::Descriptor.smarts_count c, smarts_count.keys
-    assert_equal smarts_count, result[c]
+    assert_equal [[1]], result
+    smarts = ["CC", "C", "C=C", "CO", "FF", "C1CCCC1", "NN"]
+    result = OpenTox::Algorithm::Descriptor.smarts_match c, smarts
+    assert_equal [[1, 1, 1, 0, 1, 1, 0]], result
+    smarts_count = [[10, 6, 2, 0, 2, 10, 0]]
+    result = OpenTox::Algorithm::Descriptor.smarts_count c, smarts
+    assert_equal smarts_count, result
   end
 
   def test_compound_openbabel_single
     c = OpenTox::Compound.from_smiles "CC(=O)CC(C)C#N"
     result = OpenTox::Algorithm::Descriptor.physchem c, ["Openbabel.logP"]
-    assert_equal 1, result[c].size
-    assert_equal 1.12518, result[c]["Openbabel.logP"]
+    assert_equal 1, result[0].size
+    assert_equal 1.12518, result[0][0]
   end
 
   def test_compound_cdk_single
diff --git a/test/edit_objects.rb b/test/edit_objects.rb
index e6e8d71..2b9d92a 100644
--- a/test/edit_objects.rb
+++ b/test/edit_objects.rb
@@ -9,20 +9,20 @@ class FeatureRestTest < MiniTest::Test
     f = OpenTox::Feature.new 
     f.title = "first"
     f.description = "first test description"
-    f.put
+    f.save
     
     # get object to compare first
-    f2 = OpenTox::Feature.find f.uri
+    f2 = OpenTox::Feature.find f.id
     assert_equal f.title, f2.title
     assert_equal f.description, f2.description
 
     # edit object and PUT back
     f2.title = "second"
     f2.description = f.description.reverse
-    f2.put
+    f2.save
     
     # get object to compare again
-    f3 = OpenTox::Feature.find f.uri
+    f3 = OpenTox::Feature.find f.id
     refute_equal f.title, f3.title
     refute_equal f.description, f3.description
   end
diff --git a/test/feature.rb b/test/feature.rb
index da7437a..6faf403 100644
--- a/test/feature.rb
+++ b/test/feature.rb
@@ -3,9 +3,7 @@ require_relative "setup.rb"
 class FeatureTest < MiniTest::Test
 
   def test_opentox_feature
-    @feature = OpenTox::Feature.new
-    @feature.title = "tost"
-    @feature.save
+    @feature = OpenTox::Feature.create(:title => "tost")
     assert_equal true, OpenTox::Feature.where(title: "tost").exists?, "#{@feature.id} is not accessible."
     assert_equal true, OpenTox::Feature.where(id: @feature.id).exists?, "#{@feature.id} is not accessible."
 
@@ -16,7 +14,7 @@ class FeatureTest < MiniTest::Test
     @feature2 = OpenTox::Feature.find(@feature.id)
     assert_equal "tost", @feature2[:title]
     assert_equal "tost", @feature2.title
-    assert_equal 'Feature', @feature2.type
+    assert_kind_of Feature, @feature2
 
     @feature2[:title] = "feature2"
     @feature2.save
@@ -28,8 +26,6 @@ class FeatureTest < MiniTest::Test
 
     id = @feature2.id
     @feature2.delete
-    #TODO
-    #assert_raises OpenTox::ResourceNotFoundError do
     assert_raises Mongoid::Errors::DocumentNotFound do
       OpenTox::Feature.find(id)
     end
@@ -38,17 +34,16 @@ class FeatureTest < MiniTest::Test
   def test_duplicated_features
     metadata = {
       :title => "feature duplication test",
-      :string => true,
+      :nominal => true,
       :description => "feature duplication test"
     }
-    feature = OpenTox::Feature.find_or_create_by metadata
-    dup_feature = OpenTox::Feature.find_or_create_by metadata
+    feature = NumericBioAssay.find_or_create_by metadata
+    dup_feature = NumericBioAssay.find_or_create_by metadata
+    assert_kind_of Feature, feature
     assert !feature.id.nil?, "No Feature ID in #{feature.inspect}"
     assert !feature.id.nil?, "No Feature ID in #{dup_feature.inspect}"
     assert_equal feature.id, dup_feature.id
     feature.delete
-    #TODO
-    #assert_raises OpenTox::ResourceNotFoundError do
     assert_raises Mongoid::Errors::DocumentNotFound do
       OpenTox::Feature.find(feature.id)
     end
@@ -57,6 +52,14 @@ class FeatureTest < MiniTest::Test
     end
   end
 
-end
-
+  def test_smarts_feature
+    feature = Smarts.find_or_create_by(:smarts => "CN")
+    assert feature.smarts, "CN"
+    assert_kind_of Smarts, feature
+    feature.smarts = 'cc'
+    assert feature.smarts, "cc"
+    original = Feature.where(:title => 'CN').first
+    assert original.smarts, "CN"
+  end
 
+end
diff --git a/test/fminer-long.rb b/test/fminer-long.rb
new file mode 100644
index 0000000..e396145
--- /dev/null
+++ b/test/fminer-long.rb
@@ -0,0 +1,36 @@
+require_relative "setup.rb"
+
+class FminerTest < MiniTest::Test
+
+  def test_fminer_multicell
+    # TODO aborts, probably fminer
+    dataset = OpenTox::MeasuredDataset.new 
+    #multi_cell_call.csv
+    dataset.upload File.join(DATA_DIR,"multi_cell_call.csv")
+    feature_dataset = OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset)#, :min_frequency => 15)
+    dataset.delete
+    feature_dataset.delete
+  end
+
+  def test_fminer_isscan
+    dataset = OpenTox::MeasuredDataset.new 
+    dataset.upload File.join(DATA_DIR,"ISSCAN-multi.csv")
+    feature_dataset = OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset)#, :min_frequency => 15)
+    assert_equal feature_dataset.compounds.size, dataset.compounds.size
+    p feature_dataset
+    dataset.delete
+    feature_dataset.delete
+  end
+
+  def test_fminer_kazius
+    dataset = OpenTox::MeasuredDataset.from_csv_file File.join(DATA_DIR,"kazius.csv")
+    feature_dataset = OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset, :min_frequency => 200)
+    #feature_dataset = OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset)#, :min_frequency => 15)
+    assert_equal feature_dataset.compounds.size, dataset.compounds.size
+    p feature_dataset.compounds.size
+    p feature_dataset.features.size
+    dataset.delete
+    feature_dataset.delete
+  end
+
+end
diff --git a/test/fminer.rb b/test/fminer.rb
index 663aa03..5e8bc6e 100644
--- a/test/fminer.rb
+++ b/test/fminer.rb
@@ -8,12 +8,6 @@ class FminerTest < MiniTest::Test
     refute_nil dataset.id
 
     feature_dataset = OpenTox::Algorithm::Fminer.bbrc :dataset => dataset
-    # TODO do we need tasks here?
-    #task = OpenTox::Algorithm::Fminer.bbrc :dataset => dataset#.uri
-    #task.wait
-    #feature_dataset = task.result
-    #p task.code
-    #p feature_dataset.features
     assert_equal dataset.compounds.size, feature_dataset.compounds.size
     assert_equal 54, feature_dataset.features.size
     assert_equal '[#6&A]-[#6&A]-[#6&A]=[#6&A]', feature_dataset.features.first.title
diff --git a/test/kazius-fminer.rb b/test/kazius-fminer.rb
new file mode 100644
index 0000000..a356814
--- /dev/null
+++ b/test/kazius-fminer.rb
@@ -0,0 +1,17 @@
+require_relative "setup.rb"
+
+class FminerTest < MiniTest::Test
+
+  def test_fminer_kazius
+
+    dataset = OpenTox::MeasuredDataset.from_csv_file File.join(DATA_DIR,"kazius.csv")
+    feature_dataset = OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset, :min_frequency => 75)
+    #feature_dataset = OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset)#, :min_frequency => 15)
+    assert_equal feature_dataset.compounds.size, dataset.compounds.size
+    p feature_dataset.compounds.size
+    p feature_dataset.features.size
+    dataset.delete
+    feature_dataset.delete
+  end
+
+end
diff --git a/test/lazar-fminer.rb b/test/lazar-fminer.rb
index a0951c5..17a52b1 100644
--- a/test/lazar-fminer.rb
+++ b/test/lazar-fminer.rb
@@ -3,10 +3,10 @@ require_relative "setup.rb"
 class LazarFminerTest < MiniTest::Test
 
   def test_lazar_fminer
-    dataset = OpenTox::Dataset.new
+    dataset = OpenTox::MeasuredDataset.new
     dataset.upload File.join(DATA_DIR,"hamster_carcinogenicity.csv")
     model = OpenTox::Model::Lazar.create OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset)
-    feature_dataset = OpenTox::Dataset.find model.feature_dataset_id
+    feature_dataset = OpenTox::CalculatedDataset.find model.feature_dataset_id
     assert_equal dataset.compounds.size, feature_dataset.compounds.size
     assert_equal 54, feature_dataset.features.size
     feature_dataset.data_entries.each do |e|
@@ -32,7 +32,7 @@ class LazarFminerTest < MiniTest::Test
     end
 
     # make a dataset prediction
-    compound_dataset = OpenTox::Dataset.new
+    compound_dataset = OpenTox::MeasuredDataset.new
     compound_dataset.upload File.join(DATA_DIR,"EPAFHM.mini.csv")
     #assert_equal compound_dataset.uri.uri?, true
     prediction = model.predict :dataset => compound_dataset
diff --git a/test/lazar-long.rb b/test/lazar-long.rb
index 68b300f..8f59ec3 100644
--- a/test/lazar-long.rb
+++ b/test/lazar-long.rb
@@ -3,25 +3,18 @@ require_relative "setup.rb"
 class LazarExtendedTest < MiniTest::Test
 
   def test_lazar_bbrc_ham_minfreq
-    dataset = OpenTox::Dataset.new 
+    dataset = OpenTox::MeasuredDataset.new 
     dataset.upload File.join(DATA_DIR,"hamster_carcinogenicity.csv")
-    assert_equal dataset.uri.uri?, true
-    model_uri = OpenTox::Model::Lazar.create :dataset_uri => dataset.uri, :feature_generation_uri => File.join($algorithm[:uri],"fminer","bbrc"), :min_frequency => 5
-    assert_equal model_uri.uri?, true
-    model = OpenTox::Model::Lazar.new model_uri
-    assert_equal model.uri.uri?, true
-    feature_dataset_uri = model[RDF::OT.featureDataset]
-    feature_dataset = OpenTox::Dataset.new feature_dataset_uri 
+    model = OpenTox::Model::Lazar.create OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset, :min_frequency => 5)
+    feature_dataset = OpenTox::CalculatedDataset.find model.feature_dataset_id
     assert_equal dataset.compounds.size, feature_dataset.compounds.size
     assert_equal 41, feature_dataset.features.size
-    assert_equal '[#7&A]-[#6&A]=[#7&A]', OpenTox::Feature.new(feature_dataset.features.first.uri).title
+    assert_equal '[#7&A]-[#6&A]=[#7&A]', feature_dataset.features.first.title
     compound = OpenTox::Compound.from_inchi("InChI=1S/C6H6/c1-2-4-6-5-3-1/h1-6H")
-    prediction_uri = model.run :compound_uri => compound.uri
-    prediction_dataset = OpenTox::Dataset.new prediction_uri
-    assert_equal prediction_dataset.uri.uri?, true
-    prediction = prediction_dataset.predictions.select{|p| p[:compound].uri == compound.uri}.first
-    assert_equal "false", prediction[:value]
-    assert_equal 0.12380952380952381, prediction[:confidence]
+    prediction_dataset = model.predict :compound => compound
+    prediction = prediction_dataset.data_entries.first
+    assert_equal "false", prediction.first
+    assert_equal 0.12380952380952381, prediction.last
     dataset.delete
     model.delete
     feature_dataset.delete
@@ -29,24 +22,20 @@ class LazarExtendedTest < MiniTest::Test
   end
 
   def test_lazar_bbrc_large_ds
-    dataset = OpenTox::Dataset.new 
+    # TODO fminer crashes with these settings
+    dataset = OpenTox::MeasuredDataset.new 
     dataset.upload File.join(DATA_DIR,"multi_cell_call_no_dup.csv")
-    assert_equal dataset.uri.uri?, true
-    model_uri = OpenTox::Model::Lazar.create :dataset_uri => dataset.uri, :feature_generation_uri => File.join($algorithm[:uri],"fminer","bbrc"), :min_frequency => 75
-    assert_equal model_uri.uri?, true
-    model = OpenTox::Model::Lazar.new model_uri
-    assert_equal model.uri.uri?, true
-    feature_dataset_uri = model[RDF::OT.featureDataset]
-    feature_dataset = OpenTox::Dataset.new feature_dataset_uri 
+    feature_dataset = OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset)#, :min_frequency => 15)
+    model = OpenTox::Model::Lazar.create feature_dataset
+    model.save
+    p model.id
+    feature_dataset = OpenTox::CalculatedDataset.find model.feature_dataset_id
     assert_equal dataset.compounds.size, feature_dataset.compounds.size
     assert_equal 52, feature_dataset.features.size
-    assert_equal '[#17&A]-[#6&A]', OpenTox::Feature.new(feature_dataset.features.first.uri).title
+    assert_equal '[#17&A]-[#6&A]', feature_dataset.features.first.title
     compound = OpenTox::Compound.from_inchi("InChI=1S/C10H9NO2S/c1-8-2-4-9(5-3-8)13-6-10(12)11-7-14/h2-5H,6H2,1H3")
-    prediction_uri = model.run :compound_uri => compound.uri
-    prediction_dataset = OpenTox::Dataset.new prediction_uri
-    assert_equal prediction_dataset.uri.uri?, true
-    prediction = prediction_dataset.predictions.select{|p| p[:compound].uri == compound.uri}.first
-    assert_equal "0", prediction[:value]
+    prediction_dataset = model.predict :compound => compound
+    prediction = prediction_dataset.data_entries.first
     assert_in_delta 0.025, prediction[:confidence], 0.001
     #assert_equal 0.025885845574483608, prediction[:confidence]
     # with compound change in training_dataset see:
@@ -58,4 +47,15 @@ class LazarExtendedTest < MiniTest::Test
     prediction_dataset.delete
   end
 
+  def test_lazar_kazius
+    # TODO find a solution for feature datasets > 16M (size limit in mongodb)
+    dataset = OpenTox::MeasuredDataset.from_csv_file File.join(DATA_DIR,"kazius.csv")
+    feature_dataset = OpenTox::Algorithm::Fminer.bbrc(:dataset => dataset, :min_frequency => 100)
+    assert_equal feature_dataset.compounds.size, dataset.compounds.size
+    model = OpenTox::Model::Lazar.create feature_dataset
+    p model.id
+    dataset.delete
+    #feature_dataset.delete
+  end
+
 end
diff --git a/test/lazar-physchem-long.rb b/test/lazar-physchem-long.rb
index 18f6bff..39e353e 100644
--- a/test/lazar-physchem-long.rb
+++ b/test/lazar-physchem-long.rb
@@ -5,18 +5,9 @@ class LazarPhyschemDescriptorTest < MiniTest::Test
   def test_lazar_pc_descriptors
 
     # check available descriptors
-    @descriptors = OpenTox::Algorithm::Descriptor.physchem_descriptors.keys
-    assert_equal 111,@descriptors.size,"wrong num physchem descriptors"
-    @descriptor_values = OpenTox::Algorithm::Descriptor.physchem_descriptor_values
-    assert_equal 356,@descriptor_values.size,"wrong num physchem descriptors"
-    sum = 0
-    [ @descriptors, @descriptor_values ].each do |desc|
-      {"Openbabel"=>16,"Cdk"=>(desc==@descriptors ? 50 : 295),"Joelib"=>45}.each do |k,v|
-        assert_equal v,desc.select{|x| x=~/^#{k}\./}.size,"wrong num #{k} descriptors"
-        sum += v
-      end
-    end
-    assert_equal (111+356),sum
+    @descriptors = OpenTox::Algorithm::Descriptor::DESCRIPTORS.keys
+    assert_equal 111,@descriptors.size,"wrong number of physchem descriptors"
+    @descriptor_values = OpenTox::Algorithm::Descriptor::DESCRIPTOR_VALUES
 
     # select descriptors for test
     @num_features_offset = 0
@@ -32,10 +23,8 @@ class LazarPhyschemDescriptorTest < MiniTest::Test
     puts "Descriptors: #{@descriptors}"
 
     # UPLOAD DATA
-    @dataset = OpenTox::Dataset.new 
-    @dataset.upload File.join(DATA_DIR,"EPAFHM.medi.csv")
-    assert_equal @dataset.uri.uri?, true
-    puts "Dataset: "+@dataset.uri
+    @dataset = OpenTox::MeasuredDataset.from_csv_file File.join(DATA_DIR,"EPAFHM.medi.csv")
+    puts "Dataset: "+@dataset.id
 
     @compound_smiles = "CC(C)(C)CN"
     @compound_inchi = "InChI=1S/C5H13N/c1-5(2,3)4-6/h4,6H2,1-3H3"
@@ -53,8 +42,8 @@ class LazarPhyschemDescriptorTest < MiniTest::Test
 
   def build_model_and_predict(precompute_feature_dataset=true)
 
-    model_params = {:dataset_uri => @dataset.uri}
-    feat_gen_uri = File.join($algorithm[:uri],"descriptor","physchem")
+    model_params = {:dataset => @dataset}
+    #feat_gen_uri = File.join($algorithm[:uri],"descriptor","physchem")
     
     if precompute_feature_dataset
       # PRECOMPUTE FEATURES
@@ -63,26 +52,46 @@ class LazarPhyschemDescriptorTest < MiniTest::Test
       f.puts File.read(File.join(DATA_DIR,"EPAFHM.medi.csv"))
       f.puts "\"#{@compound_smiles}\","
       f.close
-      d = OpenTox::Dataset.new 
-      d.upload p
-      model_params[:feature_dataset_uri] = OpenTox::Algorithm::Generic.new(feat_gen_uri).run({:dataset_uri => d.uri, :descriptors => @descriptors})
+      d = OpenTox::Dataset.from_csv_file p
+      descriptors = OpenTox::Algorithm::Descriptor.physchem(d.compounds, @descriptors)
+      #model_params[:feature_dataset_uri] = OpenTox::Algorithm::Generic.new(feat_gen_uri).run({:dataset_uri => d.uri, :descriptors => @descriptors})
     else
       model_params[:feature_generation_uri] = feat_gen_uri
       model_params[:descriptors] = @descriptors
     end
       
     # BUILD MODEL
-    model_uri = OpenTox::Model::Lazar.create model_params
-    puts "Model: "+model_uri
-    model = OpenTox::Model::Lazar.new model_uri
-    assert_equal model_uri.uri?, true
-    puts "Predicted variable: "+model.predicted_variable
+
+    #p descriptors
+    feature_dataset = OpenTox::CalculatedDataset.new
+    feature_dataset.compounds = @dataset.compounds
+    feature_dataset.data_entries = descriptors
+    feature_dataset.features = @descriptors.collect{|d| OpenTox::Feature.find_or_create_by(:title => d)}
+    feature_dataset["inchis"].each do |inchi|
+      assert_kind_of String, inchi
+    end
+    feature_dataset["feature_ids"].each do |id|
+      assert_kind_of BSON::ObjectId, id
+    end
+    feature_dataset.data_entries.each do |entry|
+      #p entry
+      assert_kind_of Array, entry
+      entry.each do |e|
+        #p e
+        assert_kind_of Float, e
+      end
+    end
+    feature_dataset.save
+    model = OpenTox::Model::Lazar.create @dataset, feature_dataset
+    #model = OpenTox::Model::Lazar.new model_uri
+    #assert_equal model_uri.uri?, true
+    #puts "Predicted variable: "+model.predicted_variable
     
     # CHECK FEATURE DATASET
-    feature_dataset_uri = model.metadata[RDF::OT.featureDataset].first
-    puts "Feature dataset: #{feature_dataset_uri}"
-    feature_dataset = OpenTox::Dataset.new(feature_dataset_uri)
-    assert_equal @dataset.compounds.size,feature_dataset.compounds.size-(precompute_feature_dataset ? 1 : 0),"num compounds in feature dataset not correct"
+    #feature_dataset_uri = model.metadata[RDF::OT.featureDataset].first
+    #puts "Feature dataset: #{feature_dataset_uri}"
+    #feature_dataset = OpenTox::Dataset.new(feature_dataset_uri)
+    assert_equal @dataset.compounds.size,feature_dataset.compounds.size,"Incorrect number of compounds in feature dataset"
     features = feature_dataset.features
     feature_titles = features.collect{|f| f.title}
     @descriptors.each do |d|
@@ -96,14 +105,17 @@ class LazarPhyschemDescriptorTest < MiniTest::Test
         assert feature_titles.include?(d),"feature not found #{d} in feature dataset #{feature_titles.inspect}"
       end
     end
-    assert_equal (@descriptors.size+@num_features_offset),features.size,"wrong num features in feature dataset"
+    assert_equal @descriptors.size,features.size,"Incorrect number of features in feature dataset"
+    #assert_equal (@descriptors.size+@num_features_offset),features.size,"wrong num features in feature dataset"
 
     # predict compound
     compound_uri = "#{$compound[:uri]}/#{@compound_inchi}"
-    prediction_uri = model.predict :compound_uri => compound_uri
-    prediction = OpenTox::Dataset.new prediction_uri
-    assert_equal prediction.uri.uri?, true
-    puts "Prediction "+prediction.uri
+    compound = OpenTox::Compound.new @compound_inchi
+    prediction = model.predict :compound => compound
+    p prediction
+    #prediction = OpenTox::Dataset.new prediction_uri
+    #assert_equal prediction.uri.uri?, true
+    #puts "Prediction "+prediction.uri
     
     # check prediction
     assert prediction.features.collect{|f| f.uri}.include?(model.predicted_variable),"prediction feature #{model.predicted_variable} not included prediction dataset #{prediction.features.collect{|f| f.uri}}"
diff --git a/test/model.rb b/test/rest/model.rb
index 027c9df..027c9df 100644
--- a/test/model.rb
+++ b/test/rest/model.rb
diff --git a/test/setup.rb b/test/setup.rb
index 3521645..9d9bc64 100644
--- a/test/setup.rb
+++ b/test/setup.rb
@@ -4,22 +4,7 @@ Bundler.require
 require 'opentox-client'
 require File.join(ENV["HOME"],".opentox","config","test.rb")
 
+include OpenTox
 TEST_DIR ||= File.expand_path(File.dirname(__FILE__))
 DATA_DIR ||= File.join(TEST_DIR,"data")
-
-unless $aa[:uri].to_s == ""
-  OpenTox::Authorization.authenticate($aa[:user], $aa[:password])
-  unauthorized_error "Failed to authenticate user \"#{$aa[:user]}\"." unless OpenTox::Authorization.is_token_valid
-end
-
-class OpenTox::Error
-  def to_s
-    s = super.to_s
-    if error_cause
-      s << "\nerror-cause:\n"
-      error_cause.is_a?(String) ?  s << error_cause : s << JSON.pretty_generate(error_cause)
-      s << "\n"
-    end
-    s
-  end
-end
+$mongo.database.drop