From 0882c2cd0de934d7377fc9d08c306be98612c88a Mon Sep 17 00:00:00 2001 From: "helma@in-silico.ch" Date: Fri, 16 Nov 2018 18:42:42 +0100 Subject: real datasets for testing, test data cleanup, Daphnia import, upper and lower similarity thresholds --- lib/validation-statistics.rb | 163 ++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 96 deletions(-) (limited to 'lib/validation-statistics.rb') diff --git a/lib/validation-statistics.rb b/lib/validation-statistics.rb index f3e3af8..8a8970e 100644 --- a/lib/validation-statistics.rb +++ b/lib/validation-statistics.rb @@ -7,79 +7,55 @@ module OpenTox # @return [Hash] def statistics self.accept_values = model.prediction_feature.accept_values - self.confusion_matrix = {:all => Array.new(accept_values.size){Array.new(accept_values.size,0)}, :without_warnings => Array.new(accept_values.size){Array.new(accept_values.size,0)}} - self.weighted_confusion_matrix = {:all => Array.new(accept_values.size){Array.new(accept_values.size,0)}, :without_warnings => Array.new(accept_values.size){Array.new(accept_values.size,0)}} - self.nr_predictions = {:all => 0,:without_warnings => 0} + self.confusion_matrix = {:all => Array.new(accept_values.size){Array.new(accept_values.size,0)}, :confidence_high => Array.new(accept_values.size){Array.new(accept_values.size,0)}, :confidence_low => Array.new(accept_values.size){Array.new(accept_values.size,0)}} + self.nr_predictions = {:all => 0,:confidence_high => 0,:confidence_low => 0} predictions.each do |cid,pred| - # TODO - # use predictions without probabilities (single neighbor)?? - # use measured majority class?? + # TODO: use measured majority class or all measurements?? if pred[:measurements].uniq.size == 1 and pred[:probabilities] m = pred[:measurements].first if pred[:value] == m - if pred[:value] == accept_values[0] - confusion_matrix[:all][0][0] += 1 - weighted_confusion_matrix[:all][0][0] += pred[:probabilities][pred[:value]] - self.nr_predictions[:all] += 1 - if pred[:warnings].empty? - confusion_matrix[:without_warnings][0][0] += 1 - weighted_confusion_matrix[:without_warnings][0][0] += pred[:probabilities][pred[:value]] - self.nr_predictions[:without_warnings] += 1 - end - elsif pred[:value] == accept_values[1] - confusion_matrix[:all][1][1] += 1 - weighted_confusion_matrix[:all][1][1] += pred[:probabilities][pred[:value]] - self.nr_predictions[:all] += 1 - if pred[:warnings].empty? - confusion_matrix[:without_warnings][1][1] += 1 - weighted_confusion_matrix[:without_warnings][1][1] += pred[:probabilities][pred[:value]] - self.nr_predictions[:without_warnings] += 1 + accept_values.each_with_index do |v,i| + if pred[:value] == v + confusion_matrix[:all][i][i] += 1 + self.nr_predictions[:all] += 1 + if pred[:confidence].match(/High/i) + confusion_matrix[:confidence_high][i][i] += 1 + self.nr_predictions[:confidence_high] += 1 + elsif pred[:confidence].match(/Low/i) + confusion_matrix[:confidence_low][i][i] += 1 + self.nr_predictions[:confidence_low] += 1 + end end end elsif pred[:value] != m - if pred[:value] == accept_values[0] - confusion_matrix[:all][0][1] += 1 - weighted_confusion_matrix[:all][0][1] += pred[:probabilities][pred[:value]] - self.nr_predictions[:all] += 1 - if pred[:warnings].empty? - confusion_matrix[:without_warnings][0][1] += 1 - weighted_confusion_matrix[:without_warnings][0][1] += pred[:probabilities][pred[:value]] - self.nr_predictions[:without_warnings] += 1 - end - elsif pred[:value] == accept_values[1] - confusion_matrix[:all][1][0] += 1 - weighted_confusion_matrix[:all][1][0] += pred[:probabilities][pred[:value]] - self.nr_predictions[:all] += 1 - if pred[:warnings].empty? - confusion_matrix[:without_warnings][1][0] += 1 - weighted_confusion_matrix[:without_warnings][1][0] += pred[:probabilities][pred[:value]] - self.nr_predictions[:without_warnings] += 1 + accept_values.each_with_index do |v,i| + if pred[:value] == v + confusion_matrix[:all][i][(i+1)%2] += 1 + self.nr_predictions[:all] += 1 + if pred[:confidence].match(/High/i) + confusion_matrix[:confidence_high][i][(i+1)%2] += 1 + self.nr_predictions[:confidence_high] += 1 + elsif pred[:confidence].match(/Low/i) + confusion_matrix[:confidence_low][i][(i+1)%2] += 1 + self.nr_predictions[:confidence_low] += 1 + end end end end end end - self.true_rate = {:all => {}, :without_warnings => {}} - self.predictivity = {:all => {}, :without_warnings => {}} + + self.true_rate = {:all => {}, :confidence_high => {}, :confidence_low => {}} + self.predictivity = {:all => {}, :confidence_high => {}, :confidence_low => {}} accept_values.each_with_index do |v,i| - [:all,:without_warnings].each do |a| + [:all,:confidence_high,:confidence_low].each do |a| self.true_rate[a][v] = confusion_matrix[a][i][i]/confusion_matrix[a][i].reduce(:+).to_f self.predictivity[a][v] = confusion_matrix[a][i][i]/confusion_matrix[a].collect{|n| n[i]}.reduce(:+).to_f end end - confidence_sum = {:all => 0, :without_warnings => 0} - [:all,:without_warnings].each do |a| - weighted_confusion_matrix[a].each do |r| - r.each do |c| - confidence_sum[a] += c - end - end - end self.accuracy = {} - self.weighted_accuracy = {} - [:all,:without_warnings].each do |a| + [:all,:confidence_high,:confidence_low].each do |a| self.accuracy[a] = (confusion_matrix[a][0][0]+confusion_matrix[a][1][1])/nr_predictions[a].to_f - self.weighted_accuracy[a] = (weighted_confusion_matrix[a][0][0]+weighted_confusion_matrix[a][1][1])/confidence_sum[a].to_f end $logger.debug "Accuracy #{accuracy}" $logger.debug "Nr Predictions #{nr_predictions}" @@ -87,9 +63,7 @@ module OpenTox { :accept_values => accept_values, :confusion_matrix => confusion_matrix, - :weighted_confusion_matrix => weighted_confusion_matrix, :accuracy => accuracy, - :weighted_accuracy => weighted_accuracy, :true_rate => self.true_rate, :predictivity => self.predictivity, :nr_predictions => nr_predictions, @@ -138,47 +112,27 @@ module OpenTox # Statistical evaluation of regression validations module RegressionStatistics + attr_accessor :x, :y + # Get statistics # @return [Hash] def statistics self.warnings = [] - self.rmse = {:all =>0,:without_warnings => 0} - self.r_squared = {:all =>0,:without_warnings => 0} - self.mae = {:all =>0,:without_warnings => 0} - self.within_prediction_interval = {:all =>0,:without_warnings => 0} - self.out_of_prediction_interval = {:all =>0,:without_warnings => 0} - x = {:all => [],:without_warnings => []} - y = {:all => [],:without_warnings => []} - self.nr_predictions = {:all =>0,:without_warnings => 0} + self.rmse = {:all =>0,:confidence_high => 0,:confidence_low => 0} + self.r_squared = {:all =>0,:confidence_high => 0,:confidence_low => 0} + self.mae = {:all =>0,:confidence_high => 0,:confidence_low => 0} + self.within_prediction_interval = {:all =>0,:confidence_high => 0,:confidence_low => 0} + self.out_of_prediction_interval = {:all =>0,:confidence_high => 0,:confidence_low => 0} + @x = {:all => [],:confidence_high => [],:confidence_low => []} + @y = {:all => [],:confidence_high => [],:confidence_low => []} + self.nr_predictions = {:all =>0,:confidence_high => 0,:confidence_low => 0} predictions.each do |cid,pred| !if pred[:value] and pred[:measurements] and !pred[:measurements].empty? - self.nr_predictions[:all] +=1 - x[:all] << pred[:measurements].median - y[:all] << pred[:value] - error = pred[:value]-pred[:measurements].median - self.rmse[:all] += error**2 - self.mae[:all] += error.abs - if pred[:prediction_interval] - if pred[:measurements].median >= pred[:prediction_interval][0] and pred[:measurements].median <= pred[:prediction_interval][1] - self.within_prediction_interval[:all] += 1 - else - self.out_of_prediction_interval[:all] += 1 - end - end - if pred[:warnings].empty? - self.nr_predictions[:without_warnings] +=1 - x[:without_warnings] << pred[:measurements].median - y[:without_warnings] << pred[:value] - error = pred[:value]-pred[:measurements].median - self.rmse[:without_warnings] += error**2 - self.mae[:without_warnings] += error.abs - if pred[:prediction_interval] - if pred[:measurements].median >= pred[:prediction_interval][0] and pred[:measurements].median <= pred[:prediction_interval][1] - self.within_prediction_interval[:without_warnings] += 1 - else - self.out_of_prediction_interval[:without_warnings] += 1 - end - end + insert_prediction pred, :all + if pred[:confidence].match(/High/i) + insert_prediction pred, :confidence_high + elsif pred[:confidence].match(/Low/i) + insert_prediction pred, :confidence_low end else trd_id = model.training_dataset_id @@ -187,10 +141,10 @@ module OpenTox $logger.debug "No training activities for #{smiles} in training dataset #{trd_id}." end end - [:all,:without_warnings].each do |a| - if x[a].size > 2 - R.assign "measurement", x[a] - R.assign "prediction", y[a] + [:all,:confidence_high,:confidence_low].each do |a| + if @x[a].size > 2 + R.assign "measurement", @x[a] + R.assign "prediction", @y[a] R.eval "r <- cor(measurement,prediction,use='pairwise')" self.r_squared[a] = R.eval("r").to_ruby**2 else @@ -209,7 +163,6 @@ module OpenTox $logger.debug "MAE #{mae}" $logger.debug "Nr predictions #{nr_predictions}" $logger.debug "#{within_prediction_interval} measurements within prediction interval" - $logger.debug "#{warnings}" save { :mae => mae, @@ -270,6 +223,24 @@ module OpenTox end worst_predictions.sort_by{|sid,p| p["distance_prediction_interval"] }.to_h end + + private + + def insert_prediction prediction, type + self.nr_predictions[type] +=1 + @x[type] << prediction[:measurements].median + @y[type] << prediction[:value] + error = prediction[:value]-prediction[:measurements].median + self.rmse[type] += error**2 + self.mae[type] += error.abs + if prediction[:prediction_interval] + if prediction[:measurements].median >= prediction[:prediction_interval][0] and prediction[:measurements].median <= prediction[:prediction_interval][1] + self.within_prediction_interval[type] += 1 + else + self.out_of_prediction_interval[type] += 1 + end + end + end end end end -- cgit v1.2.3