From bdc6b5b40437896384561d74a510560e9e592364 Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Tue, 9 Oct 2018 18:20:27 +0200 Subject: tentative random forest classification: hangs unpredictably during caret model generation/optimization for some (inorganic?) compounds. --- test/validation-classification.rb | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) (limited to 'test/validation-classification.rb') diff --git a/test/validation-classification.rb b/test/validation-classification.rb index ce06063..6b727d6 100644 --- a/test/validation-classification.rb +++ b/test/validation-classification.rb @@ -64,4 +64,50 @@ class ValidationClassificationTest < MiniTest::Test end end + def test_carcinogenicity_rf_classification + skip "Caret rf classification may run into a (endless?) loop for some compounds." + dataset = Dataset.from_csv_file "#{DATA_DIR}/multi_cell_call.csv" + algorithms = { + :prediction => { + :method => "Algorithm::Caret.rf", + }, + } + model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms + cv = ClassificationCrossValidation.create model +# cv = ClassificationCrossValidation.find "5bbc822dca626919731e2822" + puts cv.statistics + puts cv.id + + end + + def test_mutagenicity_classification_algorithms + skip "Caret rf classification may run into a (endless?) loop for some compounds." + source_feature = Feature.where(:name => "Ames test categorisation").first + target_feature = Feature.where(:name => "Mutagenicity").first + kazius = Dataset.from_sdf_file "#{DATA_DIR}/cas_4337.sdf" + hansen = Dataset.from_csv_file "#{DATA_DIR}/hansen.csv" + efsa = Dataset.from_csv_file "#{DATA_DIR}/efsa.csv" + dataset = Dataset.merge [kazius,hansen,efsa], {source_feature => target_feature}, {1 => "mutagen", 0 => "nonmutagen"} + model = Model::Lazar.create training_dataset: dataset + repeated_cv = RepeatedCrossValidation.create model + puts repeated_cv.id + repeated_cv.crossvalidations.each do |cv| + puts cv.accuracy + puts cv.confusion_matrix + end + algorithms = { + :prediction => { + :method => "Algorithm::Caret.rf", + }, + } + model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms + repeated_cv = RepeatedCrossValidation.create model + puts repeated_cv.id + repeated_cv.crossvalidations.each do |cv| + puts cv.accuracy + puts cv.confusion_matrix + end + + end + end -- cgit v1.2.3 From 9d17895ab9e8cd31e0f32e8e622e13612ea5ff77 Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Fri, 12 Oct 2018 21:58:36 +0200 Subject: validation statistic fixes --- test/validation-classification.rb | 113 -------------------------------------- 1 file changed, 113 deletions(-) delete mode 100644 test/validation-classification.rb (limited to 'test/validation-classification.rb') diff --git a/test/validation-classification.rb b/test/validation-classification.rb deleted file mode 100644 index 6b727d6..0000000 --- a/test/validation-classification.rb +++ /dev/null @@ -1,113 +0,0 @@ -require_relative "setup.rb" - -class ValidationClassificationTest < MiniTest::Test - include OpenTox::Validation - - # defaults - - def test_default_classification_crossvalidation - dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" - model = Model::Lazar.create training_dataset: dataset - cv = ClassificationCrossValidation.create model - assert cv.accuracy > 0.7, "Accuracy (#{cv.accuracy}) should be larger than 0.7, this may occur due to an unfavorable training/test set split" - assert cv.weighted_accuracy > cv.accuracy, "Weighted accuracy (#{cv.weighted_accuracy}) should be larger than accuracy (#{cv.accuracy})." - File.open("/tmp/tmp.pdf","w+"){|f| f.puts cv.probability_plot(format:"pdf")} - p `file -b /tmp/tmp.pdf` - File.open("/tmp/tmp.png","w+"){|f| f.puts cv.probability_plot(format:"png")} - p `file -b /tmp/tmp.png` - end - - # parameters - - def test_classification_crossvalidation_parameters - dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" - algorithms = { - :similarity => { :min => 0.3, }, - :descriptors => { :type => "FP3" } - } - model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms - cv = ClassificationCrossValidation.create model - params = model.algorithms - params = Hash[params.map{ |k, v| [k.to_s, v] }] # convert symbols to string - - cv.validations.each do |validation| - validation_params = validation.model.algorithms - refute_nil model.training_dataset_id - refute_nil validation.model.training_dataset_id - refute_equal model.training_dataset_id, validation.model.training_dataset_id - ["min_sim","type","prediction_feature_id"].each do |k| - assert_equal params[k], validation_params[k] - end - end - end - - # LOO - - def test_classification_loo_validation - dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" - model = Model::Lazar.create training_dataset: dataset - loo = ClassificationLeaveOneOut.create model - assert_equal 24, loo.nr_unpredicted - refute_empty loo.confusion_matrix - assert loo.accuracy > 0.77 - assert loo.weighted_accuracy > loo.accuracy, "Weighted accuracy (#{loo.weighted_accuracy}) should be larger than accuracy (#{loo.accuracy})." - end - - # repeated CV - - def test_repeated_crossvalidation - dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" - model = Model::Lazar.create training_dataset: dataset - repeated_cv = RepeatedCrossValidation.create model - repeated_cv.crossvalidations.each do |cv| - assert_operator cv.accuracy, :>, 0.7, "model accuracy < 0.7, this may happen by chance due to an unfavorable training/test set split" - end - end - - def test_carcinogenicity_rf_classification - skip "Caret rf classification may run into a (endless?) loop for some compounds." - dataset = Dataset.from_csv_file "#{DATA_DIR}/multi_cell_call.csv" - algorithms = { - :prediction => { - :method => "Algorithm::Caret.rf", - }, - } - model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms - cv = ClassificationCrossValidation.create model -# cv = ClassificationCrossValidation.find "5bbc822dca626919731e2822" - puts cv.statistics - puts cv.id - - end - - def test_mutagenicity_classification_algorithms - skip "Caret rf classification may run into a (endless?) loop for some compounds." - source_feature = Feature.where(:name => "Ames test categorisation").first - target_feature = Feature.where(:name => "Mutagenicity").first - kazius = Dataset.from_sdf_file "#{DATA_DIR}/cas_4337.sdf" - hansen = Dataset.from_csv_file "#{DATA_DIR}/hansen.csv" - efsa = Dataset.from_csv_file "#{DATA_DIR}/efsa.csv" - dataset = Dataset.merge [kazius,hansen,efsa], {source_feature => target_feature}, {1 => "mutagen", 0 => "nonmutagen"} - model = Model::Lazar.create training_dataset: dataset - repeated_cv = RepeatedCrossValidation.create model - puts repeated_cv.id - repeated_cv.crossvalidations.each do |cv| - puts cv.accuracy - puts cv.confusion_matrix - end - algorithms = { - :prediction => { - :method => "Algorithm::Caret.rf", - }, - } - model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms - repeated_cv = RepeatedCrossValidation.create model - puts repeated_cv.id - repeated_cv.crossvalidations.each do |cv| - puts cv.accuracy - puts cv.confusion_matrix - end - - end - -end -- cgit v1.2.3