From d27d53d98238ede80fc3b1a0c277ca890a84c736 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 18 Aug 2011 10:38:51 +0200 Subject: fix ROC stuff, rename weighted_auc to average_auc --- lib/predictions.rb | 57 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 16 deletions(-) (limited to 'lib/predictions.rb') diff --git a/lib/predictions.rb b/lib/predictions.rb index b71359d..bfb25da 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -254,7 +254,6 @@ module Lib return res end - # does only take the instances that are classified as into account def area_under_roc(class_index=nil) return prediction_feature_value_map( lambda{ |i| area_under_roc(i) } ) if class_index==nil @@ -268,15 +267,16 @@ module Lib tp_conf = [] fp_conf = [] (0..@predicted_values.size-1).each do |i| - if @predicted_values[i]==class_index - if @actual_values[i]==@predicted_values[i] - tp_conf.push(@confidence_values[i]) + if @predicted_values[i]!=nil + c = @confidence_values[i] * (@predicted_values[i]==class_index ? 1 : -1) + if @actual_values[i]==class_index + tp_conf << c else - fp_conf.push(@confidence_values[i]) + fp_conf << c end end end - #puts tp_conf.inspect+"\n"+fp_conf.inspect+"\n\n" + puts tp_conf.inspect+"\n"+fp_conf.inspect+"\n\n" return 0.0 if tp_conf.size == 0 return 1.0 if fp_conf.size == 0 @@ -432,22 +432,18 @@ module Lib return incorrect end - # Note: - # * (un-weighted) area under roc is computed with all __predicted__ isntances for a certain class - # * weighted weights each auc with the number of __acutal__ instances - # its like that, because its like that in weka - def weighted_area_under_roc - w_auc = weighted_measure( :area_under_roc ) + def average_area_under_roc + w_auc = average_measure( :area_under_roc ) w_auc.nan? ? 0 : w_auc end - def weighted_f_measure - return weighted_measure( :f_measure ) + def average_f_measure + return average_measure( :f_measure ) end private - # the is weighted with the number of instances for each actual class value - def weighted_measure( measure ) + # the is averaged over the number of instances for each actual class value + def average_measure( measure ) sum_instances = 0 num_instances_per_class = Array.new(@num_classes, 0) @@ -562,6 +558,35 @@ module Lib # data for (roc-)plots ################################################################################### + def get_roc_prediction_values(class_value) + + #puts "get_roc_values for class_value: "+class_value.to_s + raise "no confidence values" unless confidence_values_available? + raise "no class-value specified" if class_value==nil + + class_index = @accept_values.index(class_value) if class_value!=nil + raise "class not found "+class_value.to_s if (class_value!=nil && class_index==nil) + + c = []; tp = [] + (0..@predicted_values.size-1).each do |i| + if @predicted_values[i]!=nil + c << @confidence_values[i] * (@predicted_values[i]==class_index ? 1 : -1) + if (@actual_values[i]==class_index) + tp << 1 + else + tp << 0 + end + end + end + + # DO NOT raise exception here, maybe different validations are concated + #raise "no instance predicted as '"+class_value+"'" if p.size == 0 + + h = {:true_positives => tp, :confidence_values => c} + #puts h.inspect + return h + end + def get_prediction_values(class_value) #puts "get_roc_values for class_value: "+class_value.to_s -- cgit v1.2.3 From bad2d7444ab40a59770678c0b0e4057d5edeceef Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 18 Aug 2011 12:57:33 +0200 Subject: add confidence plots for various classification stats --- lib/predictions.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'lib/predictions.rb') diff --git a/lib/predictions.rb b/lib/predictions.rb index bfb25da..7de1751 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -587,19 +587,24 @@ module Lib return h end - def get_prediction_values(class_value) + def get_prediction_values(actual_accept_value, predicted_accept_value) #puts "get_roc_values for class_value: "+class_value.to_s raise "no confidence values" unless confidence_values_available? #raise "no class-value specified" if class_value==nil - class_index = @accept_values.index(class_value) if class_value!=nil - raise "class not found "+class_value.to_s if (class_value!=nil && class_index==nil) + actual_class_index = @accept_values.index(actual_accept_value) if actual_accept_value!=nil + raise "class not found '"+actual_accept_value.to_s+"' in "+@accept_values.inspect if (actual_accept_value!=nil && actual_class_index==nil) + + predicted_class_index = @accept_values.index(predicted_accept_value) if predicted_accept_value!=nil + raise "class not found "+predicted_accept_value.to_s+" in "+@accept_values.inspect if (predicted_accept_value!=nil && predicted_class_index==nil) c = []; p = []; a = [] (0..@predicted_values.size-1).each do |i| # NOTE: not predicted instances are ignored here - if @predicted_values[i]!=nil and (class_index==nil || @predicted_values[i]==class_index) + if @predicted_values[i]!=nil and + (predicted_class_index==nil || @predicted_values[i]==predicted_class_index) and + (actual_class_index==nil || @actual_values[i]==actual_class_index) c << @confidence_values[i] p << @predicted_values[i] a << @actual_values[i] -- cgit v1.2.3 From 3f1714c78e46f391b951b1a01adcd9badc713891 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 18 Aug 2011 13:46:10 +0200 Subject: add npv calculation, add ppv (renamed from precision) and npv to important classificaiton stats --- lib/predictions.rb | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'lib/predictions.rb') diff --git a/lib/predictions.rb b/lib/predictions.rb index 7de1751..2e90885 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -300,7 +300,11 @@ module Lib end def precision(class_index=nil) - return prediction_feature_value_map( lambda{ |i| precision(i) } ) if class_index==nil + return positive_predictive_value(class_index) + end + + def positive_predictive_value(class_index=nil) + return prediction_feature_value_map( lambda{ |i| positive_predictive_value(i) } ) if class_index==nil correct = 0 # all instances with prediction class_index that are correctly classified total = 0 # all instances with prediciton class_index @@ -312,6 +316,23 @@ module Lib return correct/total.to_f end + def negative_predictive_value(class_index=nil) + return prediction_feature_value_map( lambda{ |i| negative_predictive_value(i) } ) if class_index==nil + + correct = 0 # all instances with prediction class_index that are correctly classified + total = 0 # all instances with prediciton class_index + (0..@num_classes-1).each do |i| + if i != class_index + (0..@num_classes-1).each do |j| + correct += @confusion_matrix[j][i] if j != class_index + total += @confusion_matrix[j][i] + end + end + end + return 0 if total==0 + return correct/total.to_f + end + def recall(class_index=nil) return true_positive_rate(class_index) end -- cgit v1.2.3 From f17213fa992e3a644b33cd3a4f6778a6a79dd152 Mon Sep 17 00:00:00 2001 From: mguetlein Date: Thu, 18 Aug 2011 16:34:50 +0200 Subject: add ppv and npv to bar plot, remove debug output --- lib/predictions.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/predictions.rb') diff --git a/lib/predictions.rb b/lib/predictions.rb index 2e90885..6c0e996 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -276,7 +276,7 @@ module Lib end end end - puts tp_conf.inspect+"\n"+fp_conf.inspect+"\n\n" + #puts tp_conf.inspect+"\n"+fp_conf.inspect+"\n\n" return 0.0 if tp_conf.size == 0 return 1.0 if fp_conf.size == 0 -- cgit v1.2.3