diff options
Diffstat (limited to 'test/validation-classification.rb')
-rw-r--r-- | test/validation-classification.rb | 67 |
1 files changed, 67 insertions, 0 deletions
diff --git a/test/validation-classification.rb b/test/validation-classification.rb new file mode 100644 index 0000000..fb4c3e7 --- /dev/null +++ b/test/validation-classification.rb @@ -0,0 +1,67 @@ +require_relative "setup.rb" + +class ValidationClassificationTest < MiniTest::Test + include OpenTox::Validation + + # defaults + + def test_default_classification_crossvalidation + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + model = Model::Lazar.create training_dataset: dataset + cv = ClassificationCrossValidation.create model + assert cv.accuracy > 0.7, "Accuracy (#{cv.accuracy}) should be larger than 0.7, this may occur due to an unfavorable training/test set split" + assert cv.weighted_accuracy > cv.accuracy, "Weighted accuracy (#{cv.weighted_accuracy}) should be larger than accuracy (#{cv.accuracy})." + File.open("/tmp/tmp.pdf","w+"){|f| f.puts cv.probability_plot(format:"pdf")} + p `file -b /tmp/tmp.pdf` + File.open("/tmp/tmp.png","w+"){|f| f.puts cv.probability_plot(format:"png")} + p `file -b /tmp/tmp.png` + end + + # parameters + + def test_classification_crossvalidation_parameters + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + algorithms = { + :similarity => { :min => 0.3, }, + :descriptors => { :type => "FP3" } + } + model = Model::Lazar.create training_dataset: dataset, algorithms: algorithms + cv = ClassificationCrossValidation.create model + params = model.algorithms + params = Hash[params.map{ |k, v| [k.to_s, v] }] # convert symbols to string + + cv.validations.each do |validation| + validation_params = validation.model.algorithms + refute_nil model.training_dataset_id + refute_nil validation.model.training_dataset_id + refute_equal model.training_dataset_id, validation.model.training_dataset_id + ["min_sim","type","prediction_feature_id"].each do |k| + assert_equal params[k], validation_params[k] + end + end + end + + # LOO + + def test_classification_loo_validation + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + model = Model::Lazar.create training_dataset: dataset + loo = ClassificationLeaveOneOut.create model + assert_equal 14, loo.nr_unpredicted + refute_empty loo.confusion_matrix + assert loo.accuracy > 0.77 + assert loo.weighted_accuracy > loo.accuracy, "Weighted accuracy (#{loo.weighted_accuracy}) should be larger than accuracy (#{loo.accuracy})." + end + + # repeated CV + + def test_repeated_crossvalidation + dataset = Dataset.from_csv_file "#{DATA_DIR}/hamster_carcinogenicity.csv" + model = Model::Lazar.create training_dataset: dataset + repeated_cv = RepeatedCrossValidation.create model + repeated_cv.crossvalidations.each do |cv| + assert_operator cv.accuracy, :>, 0.7, "model accuracy < 0.7, this may happen by chance due to an unfavorable training/test set split" + end + end + +end |