diff options
author | mr <mr@mrautenberg.de> | 2011-07-15 12:21:01 +0200 |
---|---|---|
committer | mr <mr@mrautenberg.de> | 2011-07-15 12:24:15 +0200 |
commit | 46451d215e87c8009c95a894522cb44b214e0e57 (patch) | |
tree | dcdd3c6a8fc403e6f589bb0762a7efec0814445b | |
parent | 183fa2384e642374e7b51ba10aaa29c20bff549f (diff) |
crossvalidation confusion matrix multinominal
-rw-r--r-- | application.rb | 30 | ||||
-rw-r--r-- | helper.rb | 2 | ||||
-rw-r--r-- | model.rb | 10 | ||||
-rw-r--r-- | views/classification_validation.haml | 59 |
4 files changed, 60 insertions, 41 deletions
diff --git a/application.rb b/application.rb index 54aeb3d..2e058bb 100644 --- a/application.rb +++ b/application.rb @@ -291,7 +291,7 @@ post '/models' do # create a new model if CONFIG[:services]["opentox-validation"] @model.update :status => "Validating model" begin - validation = OpenTox::Crossvalidation.create( { + crossvalidation = OpenTox::Crossvalidation.create( { :algorithm_uri => lazar.metadata[OT.algorithm], :dataset_uri => lazar.parameter("dataset_uri"), :subjectid => subjectid, @@ -299,25 +299,37 @@ post '/models' do # create a new model :algorithm_params => "feature_generation_uri=#{lazar.parameter("feature_generation_uri")}" }, nil, OpenTox::SubTask.new(task,25,80)) - @model.update(:validation_uri => validation.uri) + @model.update(:validation_uri => crossvalidation.uri) LOGGER.debug "Validation URI: #{@model.validation_uri}" # create summary - validation.summary(subjectid).each do |k,v| - begin - eval "@model.update :#{k.to_s} => v" if v - rescue - eval "@model.update :#{k.to_s} => 0" + validation = crossvalidation.statistics(subjectid) + @model.update(:nr_predictions => validation.metadata[OT.numInstances].to_i - validation.metadata[OT.numUnpredicted].to_i) + if validation.metadata[OT.classificationStatistics] + @model.update(:correct_predictions => validation.metadata[OT.classificationStatistics][OT.percentCorrect].to_f) + @model.update(:confusion_matrix => validation.confusion_matrix.to_yaml) + @model.update(:weighted_area_under_roc => validation.metadata[OT.classificationStatistics][OT.weightedAreaUnderRoc].to_f) + validation.metadata[OT.classificationStatistics][OT.classValueStatistics].each do |m| + if m[OT.classValue] =~ TRUE_REGEXP + #HACK: estimate true feature value correctly + @model.update(:sensitivity => m[OT.truePositiveRate]) + @model.update(:specificity => m[OT.trueNegativeRate]) + break + end end + else + @model.update(:r_square => validation.metadata[OT.regressionStatistics][OT.rSquare].to_f) + @model.update(:root_mean_squared_error => validation.metadata[OT.regressionStatistics][OT.rootMeanSquaredError].to_f) + @model.update(:mean_absolute_error => validation.metadata[OT.regressionStatistics][OT.meanAbsoluteError].to_f) end rescue => e - @model.update :warnings => @model.warnings.to_s+"\nModel validation failed with #{e.message}." + @model.update :warnings => @model.warnings.to_s+"\nModel crossvalidation failed with #{e.message}." error "Model validation failed",e end begin @model.update :status => "Creating validation report" - validation_report_uri = validation.find_or_create_report(subjectid, OpenTox::SubTask.new(task,80,90)) #unless @model.dirty? + validation_report_uri = crossvalidation.find_or_create_report(subjectid, OpenTox::SubTask.new(task,80,90)) #unless @model.dirty? @model.update :validation_report_uri => validation_report_uri, :status => "Creating QMRF report" qmrf_report = OpenTox::Crossvalidation::QMRFReport.create(@model.uri, subjectid, OpenTox::SubTask.new(task,90,99)) @model.update(:validation_qmrf_uri => qmrf_report.uri, :status => "Completed") @@ -53,7 +53,7 @@ helpers do def sort(descriptors) features = {:activating => [], :deactivating => []} - descriptors.each do |d| + descriptors.each do |d| if d[OT.effect] =~ TRUE_REGEXP features[:activating] << {:smarts => d[OT.smarts],:p_value => d[OT.pValue]} elsif d[OT.effect] =~ FALSE_REGEXP @@ -27,14 +27,10 @@ class ToxCreateModel < Ohm::Model #attributey :validation_qmrf_task_uri attribute :validation_qmrf_uri - + attribute :confusion_matrix attribute :nr_compounds - attribute :nr_predictions - attribute :true_positives - attribute :false_positives - attribute :true_negatives - attribute :false_negatives - attribute :correct_predictions + attribute :nr_predictions + attribute :correct_predictions attribute :weighted_area_under_roc attribute :sensitivity attribute :specificity diff --git a/views/classification_validation.haml b/views/classification_validation.haml index 5649d7f..f25a321 100644 --- a/views/classification_validation.haml +++ b/views/classification_validation.haml @@ -1,33 +1,44 @@ %dt Correct predictions: %dd - = sprintf("%.2f", model.correct_predictions.to_f) if model.correct_predictions - = '%' + = sprintf("%.2f", model.correct_predictions.to_f) if model.correct_predictions + = '%' %dt - %a{:href => "http://en.wikipedia.org/wiki/Receiver_operating_characteristic", :rel => "external"} Weighted area under ROC: + %a{:href => "http://en.wikipedia.org/wiki/Receiver_operating_characteristic", :rel => "external"} Weighted area under ROC: %dd - = sprintf("%.3f", model.weighted_area_under_roc.to_f) if model.weighted_area_under_roc + = sprintf("%.3f", model.weighted_area_under_roc.to_f) if model.weighted_area_under_roc %dt - %a{:href => "http://en.wikipedia.org/wiki/Sensitivity_and_specificity", :rel => "external"} Specificity: + %a{:href => "http://en.wikipedia.org/wiki/Sensitivity_and_specificity", :rel => "external"} Specificity: %dd= sprintf("%.3f", model.specificity.to_f) if model.specificity %dt - %a{:href => "http://en.wikipedia.org/wiki/Sensitivity_and_specificity", :rel => "external"} Sensitivity: + %a{:href => "http://en.wikipedia.org/wiki/Sensitivity_and_specificity", :rel => "external"} Sensitivity: %dd= sprintf("%.3f", model.sensitivity.to_f) if model.sensitivity %dt - %a{:href => "http://en.wikipedia.org/wiki/Confusion_matrix", :rel => "external"} Confusion Matrix: -%dd - %table - %tr - %td{:colspan => 2, :rowspan => 2} - %th{:colspan => 2} Measured - %tr - %th{:bgcolor => "#CCD2DC"} active - %th{:bgcolor => "#CCD2DC"} inactive - %tr - %th{:rowspan => 2} Predicted - %th{:bgcolor => "#CCD2DC"} active - %td= model.true_positives if model.true_positives - %td= model.false_positives if model.false_positives - %tr - %th{:bgcolor => "#CCD2DC"} inactive - %td= model.false_negatives if model.false_negatives - %td= model.true_negatives if model.true_negatives + %a{:href => "http://en.wikipedia.org/wiki/Confusion_matrix", :rel => "external"} Confusion Matrix: +- if model.confusion_matrix + - cm = YAML.load(model.confusion_matrix) + %dd + %table + %tr + %td{:colspan => 2, :rowspan => 2} + %th{:colspan => (cm.size - 1)} Measured + %tr + - (1..cm[0].size-1).each do |i| + %th{:bgcolor => "#CCD2DC"} + = cm[0][i] + %tr + %th{:rowspan => (cm.size - 1)} Predicted + - (0..cm[1].size-1).each do |i| + - if i == 0 + %th{:bgcolor => "#CCD2DC"} + = cm[1][i] + - else + %td= cm[1][i] + - if cm.size > 2 + - (2..cm.size-1).each do |i| + %tr + - (0..cm[i].size-1).each do |j| + - if j == 0 + %th{:bgcolor => "#CCD2DC"} + = cm[i][j] + - else + %td= cm[i][j] |