From 9d17895ab9e8cd31e0f32e8e622e13612ea5ff77 Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Fri, 12 Oct 2018 21:58:36 +0200 Subject: validation statistic fixes --- test/classification-validation.rb | 126 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 test/classification-validation.rb (limited to 'test/classification-validation.rb') diff --git a/test/classification-validation.rb b/test/classification-validation.rb new file mode 100644 index 0000000..6ff8be0 --- /dev/null +++ b/test/classification-validation.rb @@ -0,0 +1,126 @@ +require_relative "setup.rb" + +class ValidationClassificationTest < MiniTest::Test + include OpenTox::Validation + + # defaults + + def test_default_classification_crossvalidation + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + model = Model::Lazar.create training_dataset: dataset + cv = ClassificationCrossValidation.create model + assert cv.accuracy[:without_warnings] > 0.65, "Accuracy (#{cv.accuracy[:without_warnings]}) should be larger than 0.65, this may occur due to an unfavorable training/test set split" + assert cv.weighted_accuracy[:all] > cv.accuracy[:all], "Weighted accuracy (#{cv.weighted_accuracy[:all]}) should be larger than accuracy (#{cv.accuracy[:all]})." + File.open("/tmp/tmp.pdf","w+"){|f| f.puts cv.probability_plot(format:"pdf")} + assert_match "PDF", `file -b /tmp/tmp.pdf` + File.open("/tmp/tmp.png","w+"){|f| f.puts cv.probability_plot(format:"png")} + assert_match "PNG", `file -b /tmp/tmp.png` + end + + # parameters + + def test_classification_crossvalidation_parameters + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + algorithms = { + :similarity => { :min => 0.3, }, + :descriptors => { :type => "FP3" } + } + model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms + cv = ClassificationCrossValidation.create model + params = model.algorithms + params = JSON.parse(params.to_json) # convert symbols to string + + cv.validations.each do |validation| + validation_params = validation.model.algorithms + refute_nil model.training_dataset_id + refute_nil validation.model.training_dataset_id + refute_equal model.training_dataset_id, validation.model.training_dataset_id + assert_equal params, validation_params + end + end + + # LOO + + def test_classification_loo_validation + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + model = Model::Lazar.create training_dataset: dataset + loo = ClassificationLeaveOneOut.create model + assert_equal 77, loo.nr_unpredicted + refute_empty loo.confusion_matrix + assert loo.accuracy[:without_warnings] > 0.650 + assert loo.weighted_accuracy[:all] > loo.accuracy[:all], "Weighted accuracy (#{loo.weighted_accuracy[:all]}) should be larger than accuracy (#{loo.accuracy[:all]})." + end + + # repeated CV + + def test_repeated_crossvalidation + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + model = Model::Lazar.create training_dataset: dataset + repeated_cv = RepeatedCrossValidation.create model + repeated_cv.crossvalidations.each do |cv| + assert_operator cv.accuracy[:without_warnings], :>, 0.65, "model accuracy < 0.65, this may happen by chance due to an unfavorable training/test set split" + end + end + + def test_validation_model + m = Model::Validation.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + [:endpoint,:species,:source].each do |p| + refute_empty m[p] + end + assert m.classification? + refute m.regression? + m.crossvalidations.each do |cv| + assert cv.accuracy[:without_warnings] > 0.65, "Crossvalidation accuracy (#{cv.accuracy[:without_warnings]}) should be larger than 0.65. This may happen due to an unfavorable training/test set split." + end + prediction = m.predict Compound.from_smiles("OCC(CN(CC(O)C)N=O)O") + assert_equal "false", prediction[:value] + m.delete + end + + def test_carcinogenicity_rf_classification + skip "Caret rf classification may run into a (endless?) loop for some compounds." + dataset = Dataset.from_csv_file "#{DATA_DIR}/multi_cell_call.csv" + algorithms = { + :prediction => { + :method => "Algorithm::Caret.rf", + }, + } + model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms + cv = ClassificationCrossValidation.create model +# cv = ClassificationCrossValidation.find "5bbc822dca626919731e2822" + puts cv.statistics + puts cv.id + + end + + def test_mutagenicity_classification_algorithms + skip "Caret rf classification may run into a (endless?) loop for some compounds." + source_feature = Feature.where(:name => "Ames test categorisation").first + target_feature = Feature.where(:name => "Mutagenicity").first + kazius = Dataset.from_sdf_file "#{DATA_DIR}/cas_4337.sdf" + hansen = Dataset.from_csv_file "#{DATA_DIR}/hansen.csv" + efsa = Dataset.from_csv_file "#{DATA_DIR}/efsa.csv" + dataset = Dataset.merge [kazius,hansen,efsa], {source_feature => target_feature}, {1 => "mutagen", 0 => "nonmutagen"} + model = Model::Lazar.create training_dataset: dataset + repeated_cv = RepeatedCrossValidation.create model + puts repeated_cv.id + repeated_cv.crossvalidations.each do |cv| + puts cv.accuracy + puts cv.confusion_matrix + end + algorithms = { + :prediction => { + :method => "Algorithm::Caret.rf", + }, + } + model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms + repeated_cv = RepeatedCrossValidation.create model + puts repeated_cv.id + repeated_cv.crossvalidations.each do |cv| + puts cv.accuracy + puts cv.confusion_matrix + end + + end + +end -- cgit v1.2.3 From d7d57cb83dbc78b2aade173bde5fa893ebf634fe Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Tue, 30 Oct 2018 07:50:29 +0000 Subject: regression feature names fixed --- test/classification-validation.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'test/classification-validation.rb') diff --git a/test/classification-validation.rb b/test/classification-validation.rb index 6ff8be0..856988a 100644 --- a/test/classification-validation.rb +++ b/test/classification-validation.rb @@ -9,6 +9,7 @@ class ValidationClassificationTest < MiniTest::Test dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" model = Model::Lazar.create training_dataset: dataset cv = ClassificationCrossValidation.create model + p cv assert cv.accuracy[:without_warnings] > 0.65, "Accuracy (#{cv.accuracy[:without_warnings]}) should be larger than 0.65, this may occur due to an unfavorable training/test set split" assert cv.weighted_accuracy[:all] > cv.accuracy[:all], "Weighted accuracy (#{cv.weighted_accuracy[:all]}) should be larger than accuracy (#{cv.accuracy[:all]})." File.open("/tmp/tmp.pdf","w+"){|f| f.puts cv.probability_plot(format:"pdf")} -- cgit v1.2.3 From d61f78093f4ddf03c27a2c8ae0bab9c1f10c80f5 Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Tue, 30 Oct 2018 17:26:59 +0100 Subject: tests fixed --- test/classification-validation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/classification-validation.rb') diff --git a/test/classification-validation.rb b/test/classification-validation.rb index 856988a..85db8ba 100644 --- a/test/classification-validation.rb +++ b/test/classification-validation.rb @@ -9,7 +9,6 @@ class ValidationClassificationTest < MiniTest::Test dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" model = Model::Lazar.create training_dataset: dataset cv = ClassificationCrossValidation.create model - p cv assert cv.accuracy[:without_warnings] > 0.65, "Accuracy (#{cv.accuracy[:without_warnings]}) should be larger than 0.65, this may occur due to an unfavorable training/test set split" assert cv.weighted_accuracy[:all] > cv.accuracy[:all], "Weighted accuracy (#{cv.weighted_accuracy[:all]}) should be larger than accuracy (#{cv.accuracy[:all]})." File.open("/tmp/tmp.pdf","w+"){|f| f.puts cv.probability_plot(format:"pdf")} @@ -68,6 +67,7 @@ class ValidationClassificationTest < MiniTest::Test [:endpoint,:species,:source].each do |p| refute_empty m[p] end + puts m.to_json assert m.classification? refute m.regression? m.crossvalidations.each do |cv| -- cgit v1.2.3 From 3a9c9332b660d35720ad4fa1f55ee0883e53aecd Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Fri, 2 Nov 2018 20:34:44 +0100 Subject: warnings fixed, cleanup --- test/classification-validation.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'test/classification-validation.rb') diff --git a/test/classification-validation.rb b/test/classification-validation.rb index 85db8ba..302b2c8 100644 --- a/test/classification-validation.rb +++ b/test/classification-validation.rb @@ -1,12 +1,13 @@ require_relative "setup.rb" -class ValidationClassificationTest < MiniTest::Test +class ClassificationValidationTest < MiniTest::Test include OpenTox::Validation # defaults def test_default_classification_crossvalidation - dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + #dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + dataset = Dataset.from_csv_file "#{DATA_DIR}/multi_cell_call.csv" model = Model::Lazar.create training_dataset: dataset cv = ClassificationCrossValidation.create model assert cv.accuracy[:without_warnings] > 0.65, "Accuracy (#{cv.accuracy[:without_warnings]}) should be larger than 0.65, this may occur due to an unfavorable training/test set split" @@ -45,7 +46,6 @@ class ValidationClassificationTest < MiniTest::Test dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" model = Model::Lazar.create training_dataset: dataset loo = ClassificationLeaveOneOut.create model - assert_equal 77, loo.nr_unpredicted refute_empty loo.confusion_matrix assert loo.accuracy[:without_warnings] > 0.650 assert loo.weighted_accuracy[:all] > loo.accuracy[:all], "Weighted accuracy (#{loo.weighted_accuracy[:all]}) should be larger than accuracy (#{loo.accuracy[:all]})." -- cgit v1.2.3 From 0882c2cd0de934d7377fc9d08c306be98612c88a Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Fri, 16 Nov 2018 18:42:42 +0100 Subject: real datasets for testing, test data cleanup, Daphnia import, upper and lower similarity thresholds --- test/classification-validation.rb | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'test/classification-validation.rb') diff --git a/test/classification-validation.rb b/test/classification-validation.rb index 302b2c8..33f0353 100644 --- a/test/classification-validation.rb +++ b/test/classification-validation.rb @@ -6,12 +6,10 @@ class ClassificationValidationTest < MiniTest::Test # defaults def test_default_classification_crossvalidation - #dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" - dataset = Dataset.from_csv_file "#{DATA_DIR}/multi_cell_call.csv" + dataset = Dataset.from_csv_file File.join(Download::DATA,"Carcinogenicity-Rodents.csv") model = Model::Lazar.create training_dataset: dataset cv = ClassificationCrossValidation.create model - assert cv.accuracy[:without_warnings] > 0.65, "Accuracy (#{cv.accuracy[:without_warnings]}) should be larger than 0.65, this may occur due to an unfavorable training/test set split" - assert cv.weighted_accuracy[:all] > cv.accuracy[:all], "Weighted accuracy (#{cv.weighted_accuracy[:all]}) should be larger than accuracy (#{cv.accuracy[:all]})." + assert cv.accuracy[:all] > 0.65, "Accuracy (#{cv.accuracy[:all]}) should be larger than 0.65, this may occur due to an unfavorable training/test set split" File.open("/tmp/tmp.pdf","w+"){|f| f.puts cv.probability_plot(format:"pdf")} assert_match "PDF", `file -b /tmp/tmp.pdf` File.open("/tmp/tmp.png","w+"){|f| f.puts cv.probability_plot(format:"png")} @@ -23,13 +21,14 @@ class ClassificationValidationTest < MiniTest::Test def test_classification_crossvalidation_parameters dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" algorithms = { - :similarity => { :min => 0.3, }, + :similarity => { :min => [0.9,0.8] }, :descriptors => { :type => "FP3" } } model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms cv = ClassificationCrossValidation.create model params = model.algorithms params = JSON.parse(params.to_json) # convert symbols to string + p cv cv.validations.each do |validation| validation_params = validation.model.algorithms @@ -47,8 +46,7 @@ class ClassificationValidationTest < MiniTest::Test model = Model::Lazar.create training_dataset: dataset loo = ClassificationLeaveOneOut.create model refute_empty loo.confusion_matrix - assert loo.accuracy[:without_warnings] > 0.650 - assert loo.weighted_accuracy[:all] > loo.accuracy[:all], "Weighted accuracy (#{loo.weighted_accuracy[:all]}) should be larger than accuracy (#{loo.accuracy[:all]})." + assert loo.accuracy[:all] > 0.650 end # repeated CV @@ -58,7 +56,7 @@ class ClassificationValidationTest < MiniTest::Test model = Model::Lazar.create training_dataset: dataset repeated_cv = RepeatedCrossValidation.create model repeated_cv.crossvalidations.each do |cv| - assert_operator cv.accuracy[:without_warnings], :>, 0.65, "model accuracy < 0.65, this may happen by chance due to an unfavorable training/test set split" + assert_operator cv.accuracy[:all], :>, 0.65, "model accuracy < 0.65, this may happen by chance due to an unfavorable training/test set split" end end @@ -71,7 +69,7 @@ class ClassificationValidationTest < MiniTest::Test assert m.classification? refute m.regression? m.crossvalidations.each do |cv| - assert cv.accuracy[:without_warnings] > 0.65, "Crossvalidation accuracy (#{cv.accuracy[:without_warnings]}) should be larger than 0.65. This may happen due to an unfavorable training/test set split." + assert cv.accuracy[:all] > 0.65, "Crossvalidation accuracy (#{cv.accuracy[:all]}) should be larger than 0.65. This may happen due to an unfavorable training/test set split." end prediction = m.predict Compound.from_smiles("OCC(CN(CC(O)C)N=O)O") assert_equal "false", prediction[:value] @@ -80,7 +78,7 @@ class ClassificationValidationTest < MiniTest::Test def test_carcinogenicity_rf_classification skip "Caret rf classification may run into a (endless?) loop for some compounds." - dataset = Dataset.from_csv_file "#{DATA_DIR}/multi_cell_call.csv" + dataset = Dataset.from_csv_file File.join(Download::DATA,"Carcinogenicity-Rodents.csv") algorithms = { :prediction => { :method => "Algorithm::Caret.rf", @@ -98,9 +96,9 @@ class ClassificationValidationTest < MiniTest::Test skip "Caret rf classification may run into a (endless?) loop for some compounds." source_feature = Feature.where(:name => "Ames test categorisation").first target_feature = Feature.where(:name => "Mutagenicity").first - kazius = Dataset.from_sdf_file "#{DATA_DIR}/cas_4337.sdf" - hansen = Dataset.from_csv_file "#{DATA_DIR}/hansen.csv" - efsa = Dataset.from_csv_file "#{DATA_DIR}/efsa.csv" + kazius = Dataset.from_sdf_file "#{Download::DATA}/parts/cas_4337.sdf" + hansen = Dataset.from_csv_file "#{Download::DATA}/parts/hansen.csv" + efsa = Dataset.from_csv_file "#{Download::DATA}/parts/efsa.csv" dataset = Dataset.merge [kazius,hansen,efsa], {source_feature => target_feature}, {1 => "mutagen", 0 => "nonmutagen"} model = Model::Lazar.create training_dataset: dataset repeated_cv = RepeatedCrossValidation.create model -- cgit v1.2.3 From b536a45cf18b070cec3f9cb8a44fdac0bfa3c58e Mon Sep 17 00:00:00 2001 From: gebele Date: Thu, 27 Jun 2019 14:08:57 +0000 Subject: fixed confidence value for cv stats; added tests --- test/classification-validation.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'test/classification-validation.rb') diff --git a/test/classification-validation.rb b/test/classification-validation.rb index 33f0353..5995c88 100644 --- a/test/classification-validation.rb +++ b/test/classification-validation.rb @@ -36,6 +36,34 @@ class ClassificationValidationTest < MiniTest::Test refute_nil validation.model.training_dataset_id refute_equal model.training_dataset_id, validation.model.training_dataset_id assert_equal params, validation_params + keys = cv.accuracy.keys + av = cv.accept_values + types = ["nr_predictions", \ + "predictivity", \ + "true_rate", \ + "confusion_matrix" + ] + types.each do |type| + keys.each do |key| + case type + when "confusion_matrix" + cv[type][key].each do |arr| + arr.each do |a| + refute_nil a + assert a > 0 + end + end + when "predictivity", "true_rate" + av.each do |v| + refute_nil cv[type][key][v] + assert cv[type][key][v] > 0 + end + else + refute_nil cv[type][key] + assert cv[type][key] > 0 + end + end + end end end -- cgit v1.2.3 From 4a8cdac587ae464244e9ae4bffc2597b6bd07101 Mon Sep 17 00:00:00 2001 From: gebele Date: Tue, 2 Jul 2019 09:04:47 +0000 Subject: adding text message for assertions; remove value check for regression test, see comment --- test/classification-validation.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'test/classification-validation.rb') diff --git a/test/classification-validation.rb b/test/classification-validation.rb index 5995c88..b913e1e 100644 --- a/test/classification-validation.rb +++ b/test/classification-validation.rb @@ -50,17 +50,17 @@ class ClassificationValidationTest < MiniTest::Test cv[type][key].each do |arr| arr.each do |a| refute_nil a - assert a > 0 + assert a > 0, "#{cv[type][key]} values should be greater than 0." end end when "predictivity", "true_rate" av.each do |v| refute_nil cv[type][key][v] - assert cv[type][key][v] > 0 + assert cv[type][key][v] > 0, "#{cv[type][key]} values should be greater than 0." end else refute_nil cv[type][key] - assert cv[type][key] > 0 + assert cv[type][key] > 0, "#{cv[type][key]} value should be greater than 0." end end end -- cgit v1.2.3