From 2dc66aef3b7932105868ee8c7d32ad975e142d1b Mon Sep 17 00:00:00 2001 From: Christoph Helma Date: Thu, 13 Oct 2016 19:48:21 +0200 Subject: compound tests fixed --- lib/caret.rb | 4 +- lib/compound.rb | 5 +- test/compound.rb | 15 +----- test/model-nanoparticle.rb | 2 +- test/validation-classification.rb | 63 ++++++++++++++++++++++ test/validation-nanoparticle.rb | 21 +++++--- test/validation.rb | 109 -------------------------------------- 7 files changed, 84 insertions(+), 135 deletions(-) create mode 100644 test/validation-classification.rb delete mode 100644 test/validation.rb diff --git a/lib/caret.rb b/lib/caret.rb index 886e2f9..df86093 100644 --- a/lib/caret.rb +++ b/lib/caret.rb @@ -30,8 +30,8 @@ module OpenTox R.eval "model <- train(activities ~ ., data = data, method = '#{method}', na.action = na.pass, allowParallel=TRUE)" rescue => e $logger.debug "R caret model creation error for:" - $logger.debug JSON.pretty_generate(dependent_variables) - $logger.debug JSON.pretty_generate(independent_variables) + $logger.debug dependent_variables + $logger.debug independent_variables return {:value => nil, :warning => "R caret model cration error."} end begin diff --git a/lib/compound.rb b/lib/compound.rb index b47364c..6c53cde 100644 --- a/lib/compound.rb +++ b/lib/compound.rb @@ -300,9 +300,8 @@ module OpenTox # Calculate molecular weight of Compound with OB and store it in object # @return [Float] molecular weight def molecular_weight - mw_feature = PhysChem.find_or_create_by(:name => "Openbabel.MW").id.to_s - calculate_properties unless properties[mw_feature] - properties[mw_feature] + mw_feature = PhysChem.find_or_create_by(:name => "Openbabel.MW") + calculate_properties([mw_feature]).first end private diff --git a/test/compound.rb b/test/compound.rb index 19f51fd..bdfb749 100644 --- a/test/compound.rb +++ b/test/compound.rb @@ -111,20 +111,9 @@ print c.sdf assert_equal 100.15888, c.molecular_weight end - def test_mg_conversions - # TODO fix! - skip - c = OpenTox::Compound.from_smiles "O" - mw = c.molecular_weight - assert_equal 18.01528, mw - assert_equal 0.8105107141417474, c.logmmol_to_mg(4.34688225631145, mw) - assert_equal 9007.64, c.mmol_to_mg(500, mw) - assert_equal 2437.9999984148976, c.logmg_to_mg(3.387033701) - end - def test_physchem c = OpenTox::Compound.from_smiles "CC(=O)CC(C)C" - assert_equal PhysChem::OPENBABEL.size, c.properties.size - assert_equal PhysChem::OPENBABEL.size, c.properties([PhysChem::OPENBABEL]).size + properties = c.calculate_properties(PhysChem.openbabel_descriptors) + assert_equal PhysChem::OPENBABEL.size, properties.size end end diff --git a/test/model-nanoparticle.rb b/test/model-nanoparticle.rb index fb81b83..6e18add 100644 --- a/test/model-nanoparticle.rb +++ b/test/model-nanoparticle.rb @@ -1,6 +1,6 @@ require_relative "setup.rb" -class NanoparticleTest < MiniTest::Test +class NanoparticleModelTest < MiniTest::Test include OpenTox::Validation def setup diff --git a/test/validation-classification.rb b/test/validation-classification.rb new file mode 100644 index 0000000..b71e427 --- /dev/null +++ b/test/validation-classification.rb @@ -0,0 +1,63 @@ +require_relative "setup.rb" + +class ValidationClassificationTest < MiniTest::Test + include OpenTox::Validation + + # defaults + + def test_default_classification_crossvalidation + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + model = Model::Lazar.create training_dataset: dataset + cv = ClassificationCrossValidation.create model + assert cv.accuracy > 0.7, "Accuracy (#{cv.accuracy}) should be larger than 0.7, this may occur due to an unfavorable training/test set split" + assert cv.weighted_accuracy > cv.accuracy, "Weighted accuracy (#{cv.weighted_accuracy}) should be larger than accuracy (#{cv.accuracy})." + end + + # parameters + + def test_classification_crossvalidation_parameters + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + algorithms = { + :similarity => { :min => 0.3, }, + :descriptors => { :type => "FP3" } + } + model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms + cv = ClassificationCrossValidation.create model + params = model.algorithms + params = Hash[params.map{ |k, v| [k.to_s, v] }] # convert symbols to string + + cv.validations.each do |validation| + validation_params = validation.model.algorithms + refute_nil model.training_dataset_id + refute_nil validation.model.training_dataset_id + refute_equal model.training_dataset_id, validation.model.training_dataset_id + ["min_sim","type","prediction_feature_id"].each do |k| + assert_equal params[k], validation_params[k] + end + end + end + + # LOO + + def test_classification_loo_validation + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + model = Model::Lazar.create training_dataset: dataset + loo = ClassificationLeaveOneOut.create model + assert_equal 14, loo.nr_unpredicted + refute_empty loo.confusion_matrix + assert loo.accuracy > 0.77 + assert loo.weighted_accuracy > loo.accuracy, "Weighted accuracy (#{loo.weighted_accuracy}) should be larger than accuracy (#{loo.accuracy})." + end + + # repeated CV + + def test_repeated_crossvalidation + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + model = Model::Lazar.create training_dataset: dataset + repeated_cv = RepeatedCrossValidation.create model + repeated_cv.crossvalidations.each do |cv| + assert_operator cv.accuracy, :>, 0.7, "model accuracy < 0.7, this may happen by chance due to an unfavorable training/test set split" + end + end + +end diff --git a/test/validation-nanoparticle.rb b/test/validation-nanoparticle.rb index 3692515..c5618e8 100644 --- a/test/validation-nanoparticle.rb +++ b/test/validation-nanoparticle.rb @@ -1,6 +1,6 @@ require_relative "setup.rb" -class NanoparticleTest < MiniTest::Test +class NanoparticleValidationTest < MiniTest::Test include OpenTox::Validation def setup @@ -24,8 +24,11 @@ class NanoparticleTest < MiniTest::Test def test_validate_pls_nanoparticle_model algorithms = { - :descriptors => { :types => ["P-CHEM"] }, - :prediction => {:parameters => 'pls' }, + :descriptors => { + :method => "properties", + :categories => ["P-CHEM"] + }, + :prediction => {:method => 'Algorithm::Caret.pls' }, } model = Model::Lazar.create prediction_feature: @prediction_feature, training_dataset: @training_dataset, algorithms: algorithms assert_equal "pls", model.algorithms[:prediction][:parameters] @@ -39,12 +42,15 @@ class NanoparticleTest < MiniTest::Test def test_validate_proteomics_pls_nanoparticle_model algorithms = { - :descriptors => { :types => ["Proteomics"] }, - :prediction => { :parameters => 'pls' } + :descriptors => { + :method => "properties", + :categories => ["Proteomics"] + }, + :prediction => {:method => 'Algorithm::Caret.pls' }, } model = Model::Lazar.create prediction_feature: @prediction_feature, training_dataset: @training_dataset, algorithms: algorithms assert_equal "pls", model.algorithms[:prediction][:parameters] - assert_equal "Algorithm::Caret.regression", model.algorithms[:prediction][:method] + assert_equal "Algorithm::Caret.pls", model.algorithms[:prediction][:method] cv = CrossValidation.create model p cv.rmse p cv.r_squared @@ -55,7 +61,8 @@ class NanoparticleTest < MiniTest::Test def test_validate_all_default_nanoparticle_model algorithms = { :descriptors => { - :types => ["Proteomics","P-CHEM"] + :method => "properties", + :categories => ["Proteomics","P-CHEM"] }, } model = Model::Lazar.create prediction_feature: @prediction_feature, training_dataset: @training_dataset, algorithms: algorithms diff --git a/test/validation.rb b/test/validation.rb deleted file mode 100644 index 03adf69..0000000 --- a/test/validation.rb +++ /dev/null @@ -1,109 +0,0 @@ -require_relative "setup.rb" - -class ValidationTest < MiniTest::Test - include OpenTox::Validation - - # defaults - - def test_default_classification_crossvalidation - dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" - model = Model::Lazar.create training_dataset: dataset - cv = ClassificationCrossValidation.create model - assert cv.accuracy > 0.7, "Accuracy (#{cv.accuracy}) should be larger than 0.7, this may occur due to an unfavorable training/test set split" - assert cv.weighted_accuracy > cv.accuracy, "Weighted accuracy (#{cv.weighted_accuracy}) should be larger than accuracy (#{cv.accuracy})." - end - - def test_default_regression_crossvalidation - dataset = Dataset.from_csv_file "#{DATA_DIR}/EPAFHM.medi_log10.csv" - model = Model::Lazar.create training_dataset: dataset - cv = RegressionCrossValidation.create model - assert cv.rmse < 1.5, "RMSE #{cv.rmse} should be smaller than 1.5, this may occur due to an unfavorable training/test set split" - assert cv.mae < 1, "MAE #{cv.mae} should be smaller than 1, this may occur due to an unfavorable training/test set split" - end - - # parameters - - def test_classification_crossvalidation_parameters - dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" - algorithms = { - :similarity => { :min => 0.3, }, - :descriptors => { :type => "FP3" } - } - model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms - cv = ClassificationCrossValidation.create model - params = model.algorithms - params = Hash[params.map{ |k, v| [k.to_s, v] }] # convert symbols to string - - cv.validations.each do |validation| - validation_params = validation.model.algorithms - refute_nil model.training_dataset_id - refute_nil validation.model.training_dataset_id - refute_equal model.training_dataset_id, validation.model.training_dataset_id - ["min_sim","type","prediction_feature_id"].each do |k| - assert_equal params[k], validation_params[k] - end - end - end - - def test_regression_crossvalidation_params - dataset = Dataset.from_csv_file "#{DATA_DIR}/EPAFHM.medi_log10.csv" - algorithms = { - :prediction => { :method => "OpenTox::Algorithm::Regression.weighted_average" }, - :descriptors => { :type => "MACCS", }, - :similarity => {:min => 0.7} - } - model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms - assert_equal algorithms[:descriptors][:type], model.algorithms[:descriptors][:type] - cv = RegressionCrossValidation.create model - cv.validation_ids.each do |vid| - model = Model::Lazar.find(Validation.find(vid).model_id) - assert_equal algorithms[:descriptors][:type], model.algorithms[:descriptors][:type] - assert_equal algorithms[:similarity][:min], model.algorithms[:similarity][:min] - refute_nil model.training_dataset_id - refute_equal dataset.id, model.training_dataset_id - end - - refute_nil cv.rmse - refute_nil cv.mae - end - - def test_physchem_regression_crossvalidation - skip # TODO: fix - training_dataset = OpenTox::Dataset.from_csv_file File.join(DATA_DIR,"EPAFHM.medi_log10.csv") - model = Model::Lazar.create(training_dataset.features.first, training_dataset, :prediction_algorithm => "OpenTox::Algorithm::Regression.local_physchem_regression") - cv = RegressionCrossValidation.create model - refute_nil cv.rmse - refute_nil cv.mae - end - - # LOO - - def test_classification_loo_validation - dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" - model = Model::Lazar.create training_dataset: dataset - loo = ClassificationLeaveOneOut.create model - assert_equal 14, loo.nr_unpredicted - refute_empty loo.confusion_matrix - assert loo.accuracy > 0.77 - assert loo.weighted_accuracy > loo.accuracy, "Weighted accuracy (#{loo.weighted_accuracy}) should be larger than accuracy (#{loo.accuracy})." - end - - def test_regression_loo_validation - dataset = OpenTox::Dataset.from_csv_file File.join(DATA_DIR,"EPAFHM.medi_log10.csv") - model = Model::Lazar.create training_dataset: dataset - loo = RegressionLeaveOneOut.create model - assert loo.r_squared > 0.34, "R^2 (#{loo.r_squared}) should be larger than 0.034" - end - - # repeated CV - - def test_repeated_crossvalidation - dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" - model = Model::Lazar.create training_dataset: dataset - repeated_cv = RepeatedCrossValidation.create model - repeated_cv.crossvalidations.each do |cv| - assert_operator cv.accuracy, :>, 0.7, "model accuracy < 0.7, this may happen by chance due to an unfavorable training/test set split" - end - end - -end -- cgit v1.2.3