diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/merge.rb | 10 | ||||
| -rwxr-xr-x | lib/ot_predictions.rb | 222 | ||||
| -rw-r--r-- | lib/prediction_data.rb | 297 | ||||
| -rwxr-xr-x | lib/predictions.rb | 201 | ||||
| -rwxr-xr-x | lib/validation_db.rb | 9 |
5 files changed, 489 insertions, 250 deletions
diff --git a/lib/merge.rb b/lib/merge.rb index f30a3c1..bc6e1a7 100644 --- a/lib/merge.rb +++ b/lib/merge.rb @@ -31,6 +31,11 @@ module Lib return merge_count(object)>1 end + def self.merge_count( object ) + @@merge_count[object] = 1 if @@merge_count[object]==nil + return @@merge_count[object] + end + def self.merge_objects( object1, object2 ) raise "classes not equal : "+object1.class.to_s+" != "+object2.class.to_s if object1.class != object2.class object_class = object1.class @@ -137,11 +142,6 @@ module Lib {:value => value, :variance => variance } end - def self.merge_count( object ) - @@merge_count[object] = 1 if @@merge_count[object]==nil - return @@merge_count[object] - end - def self.set_merge_count(object, merge_count) @@merge_count[object] = merge_count end diff --git a/lib/ot_predictions.rb b/lib/ot_predictions.rb index 335fe84..3be845b 100755 --- a/lib/ot_predictions.rb +++ b/lib/ot_predictions.rb @@ -1,228 +1,28 @@ +require "lib/prediction_data.rb" require "lib/predictions.rb" module Lib class OTPredictions < Predictions - CHECK_VALUES = ENV['RACK_ENV'] =~ /debug|test/ - + def initialize(data, compounds=nil) + raise unless data.is_a?(Hash) + super(data) + @compounds = compounds + end + def identifier(instance_index) - return compound(instance_index) + compound(instance_index) end def compound(instance_index) - return @compounds[instance_index] - end - - def initialize( feature_type, test_dataset_uris, test_target_dataset_uris, - prediction_feature, prediction_dataset_uris, predicted_variables, predicted_confidences, subjectid=nil, task=nil) - - test_dataset_uris = [test_dataset_uris] unless test_dataset_uris.is_a?(Array) - test_target_dataset_uris = [test_target_dataset_uris] unless test_target_dataset_uris.is_a?(Array) - prediction_dataset_uris = [prediction_dataset_uris] unless prediction_dataset_uris.is_a?(Array) - predicted_variables = [predicted_variables] unless predicted_variables.is_a?(Array) - predicted_confidences = [predicted_confidences] unless predicted_confidences.is_a?(Array) - LOGGER.debug "loading prediction -- test-dataset: "+test_dataset_uris.inspect - LOGGER.debug "loading prediction -- test-target-datset: "+test_target_dataset_uris.inspect - LOGGER.debug "loading prediction -- prediction-dataset: "+prediction_dataset_uris.inspect - LOGGER.debug "loading prediction -- predicted_variable: "+predicted_variables.inspect - LOGGER.debug "loading prediction -- predicted_confidence: "+predicted_confidences.inspect - LOGGER.debug "loading prediction -- prediction_feature: "+prediction_feature.to_s - raise "prediction_feature missing" unless prediction_feature - - @compounds = [] - all_predicted_values = [] - all_actual_values = [] - all_confidence_values = [] - accept_values = nil - - if task - task_step = 100 / (test_dataset_uris.size*2 + 1) - task_status = 0 - end - - test_dataset_uris.size.times do |i| - - test_dataset_uri = test_dataset_uris[i] - test_target_dataset_uri = test_target_dataset_uris[i] - prediction_dataset_uri = prediction_dataset_uris[i] - predicted_variable = predicted_variables[i] - predicted_confidence = predicted_confidences[i] - - predicted_variable=prediction_feature if predicted_variable==nil - - test_dataset = Lib::DatasetCache.find test_dataset_uri,subjectid - raise "test dataset not found: '"+test_dataset_uri.to_s+"'" unless test_dataset - - if test_target_dataset_uri == nil || test_target_dataset_uri.strip.size==0 || test_target_dataset_uri==test_dataset_uri - test_target_dataset_uri = test_dataset_uri - test_target_dataset = test_dataset - raise "prediction_feature not found in test_dataset, specify a test_target_dataset\n"+ - "prediction_feature: '"+prediction_feature.to_s+"'\n"+ - "test_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ - "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil - else - test_target_dataset = Lib::DatasetCache.find test_target_dataset_uri,subjectid - raise "test target datset not found: '"+test_target_dataset_uri.to_s+"'" unless test_target_dataset - if CHECK_VALUES - test_dataset.compounds.each do |c| - raise "test compound not found on test class dataset "+c.to_s unless test_target_dataset.compounds.include?(c) - end - end - raise "prediction_feature not found in test_target_dataset\n"+ - "prediction_feature: '"+prediction_feature.to_s+"'\n"+ - "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ - "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil - end - - compounds = test_dataset.compounds - LOGGER.debug "test dataset size: "+compounds.size.to_s - raise "test dataset is empty "+test_dataset_uri.to_s unless compounds.size>0 - - if feature_type=="classification" - av = test_target_dataset.accept_values(prediction_feature) - raise "'"+OT.acceptValue.to_s+"' missing/invalid for feature '"+prediction_feature.to_s+"' in dataset '"+ - test_target_dataset_uri.to_s+"', acceptValues are: '"+av.inspect+"'" if av==nil or av.length<2 - if accept_values==nil - accept_values=av - else - raise "accept values (in folds) differ "+av.inspect+" != "+accept_values.inspect if av!=accept_values - end - end - - actual_values = [] - compounds.each do |c| - case feature_type - when "classification" - actual_values << classification_val(test_target_dataset, c, prediction_feature, accept_values) - when "regression" - actual_values << regression_val(test_target_dataset, c, prediction_feature) - end - end - task.progress( task_status += task_step ) if task # loaded actual values - - prediction_dataset = Lib::DatasetCache.find prediction_dataset_uri,subjectid - raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset - - # allow missing prediction feature if there are no compounds in the prediction dataset - raise "predicted_variable not found in prediction_dataset\n"+ - "predicted_variable '"+predicted_variable.to_s+"'\n"+ - "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+ - "available features are: "+prediction_dataset.features.inspect if prediction_dataset.features.keys.index(predicted_variable)==nil and prediction_dataset.compounds.size>0 - raise "predicted_confidence not found in prediction_dataset\n"+ - "predicted_confidence '"+predicted_confidence.to_s+"'\n"+ - "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+ - "available features are: "+prediction_dataset.features.inspect if predicted_confidence and prediction_dataset.features.keys.index(predicted_confidence)==nil and prediction_dataset.compounds.size>0 - - raise "more predicted than test compounds, #test: "+compounds.size.to_s+" < #prediction: "+ - prediction_dataset.compounds.size.to_s+", test-dataset: "+test_dataset_uri.to_s+", prediction-dataset: "+ - prediction_dataset_uri if compounds.size < prediction_dataset.compounds.size - if CHECK_VALUES - prediction_dataset.compounds.each do |c| - raise "predicted compound not found in test dataset:\n"+c+"\ntest-compounds:\n"+ - compounds.collect{|c| c.to_s}.join("\n") if compounds.index(c)==nil - end - end - - predicted_values = [] - confidence_values = [] - count = 0 - compounds.each do |c| - if prediction_dataset.compounds.index(c)==nil - predicted_values << nil - confidence_values << nil - else - case feature_type - when "classification" - predicted_values << classification_val(prediction_dataset, c, predicted_variable, accept_values) - when "regression" - predicted_values << regression_val(prediction_dataset, c, predicted_variable) - end - if predicted_confidence - confidence_values << confidence_val(prediction_dataset, c, predicted_confidence) - else - confidence_values << nil - end - end - count += 1 - end - @compounds += compounds - all_predicted_values += predicted_values - all_actual_values += actual_values - all_confidence_values += confidence_values - - task.progress( task_status += task_step ) if task # loaded predicted values and confidence - end - - super(all_predicted_values, all_actual_values, all_confidence_values, feature_type, accept_values) - raise "illegal num compounds "+num_info if @compounds.size != @predicted_values.size - task.progress(100) if task # done with the mathmatics - end - - private - def regression_val(dataset, compound, feature) - v = value(dataset, compound, feature) - begin - v = v.to_f unless v==nil or v.is_a?(Numeric) - v - rescue - LOGGER.warn "no numeric value for regression: '"+v.to_s+"'" - nil - end - end - - def confidence_val(dataset, compound, confidence) - v = value(dataset, compound, confidence) - begin - v = v.to_f unless v==nil or v.is_a?(Numeric) - v - rescue - LOGGER.warn "no numeric value for confidence '"+v.to_s+"'" - nil - end - end - - def classification_val(dataset, compound, feature, accept_values) - v = value(dataset, compound, feature) - i = accept_values.index(v.to_s) - raise "illegal class_value of prediction (value is '"+v.to_s+"'), accept values are "+ - accept_values.inspect unless v==nil or i!=nil - i + @compounds[instance_index] end - def value(dataset, compound, feature) - return nil if dataset.data_entries[compound]==nil - if feature==nil - v = dataset.data_entries[compound].values[0] - else - v = dataset.data_entries[compound][feature] - end - return nil if v==nil - raise "no array "+v.class.to_s+" : '"+v.to_s+"'" unless v.is_a?(Array) - if v.size>1 - v.uniq! - if v.size>1 - v = nil - LOGGER.warn "not yet implemented: multiple non-equal values "+compound.to_s+" "+v.inspect - else - v = v[0] - end - elsif v.size==1 - v = v[0] - else - v = nil - end - raise "array" if v.is_a?(Array) - v = nil if v.to_s.size==0 - v - end - - public - def compute_stats - + def compute_stats() res = {} - case @feature_type + case feature_type when "classification" (Validation::VAL_CLASS_PROPS).each{ |s| res[s] = send(s)} when "regression" diff --git a/lib/prediction_data.rb b/lib/prediction_data.rb new file mode 100644 index 0000000..42da5fc --- /dev/null +++ b/lib/prediction_data.rb @@ -0,0 +1,297 @@ + +module Lib + + class PredictionData + + CHECK_VALUES = ENV['RACK_ENV'] =~ /debug|test/ + + def self.filter_data( data, compounds, min_confidence, min_num_predictions, max_num_predictions, prediction_index=nil ) + + raise "cannot filter anything, no confidence values available" if data[:confidence_values][0]==nil + raise OpenTox::BadRequestError.new "please specify either min_confidence or max_num_predictions" if + (min_confidence!=nil and max_num_predictions!=nil) || (min_confidence==nil and max_num_predictions==nil) + raise OpenTox::BadRequestError.new "min_num_predictions only valid for min_confidence" if + (min_confidence==nil and min_num_predictions!=nil) + min_num_predictions = 0 if min_num_predictions==nil + + LOGGER.debug("filtering predictions, conf:'"+min_confidence.to_s+"' min_num_predictions: '"+ + min_num_predictions.to_s+"' max_num_predictions: '"+max_num_predictions.to_s+"' ") + #LOGGER.debug("to filter:\nconf: "+data[:confidence_values].inspect) + + orig_size = data[:predicted_values].size + valid_indices = [] + data[:confidence_values].size.times do |i| + next if prediction_index!=nil and prediction_index!=data[:predicted_values][i] + valid = false + if min_confidence!=nil + valid = (valid_indices.size<=min_num_predictions or + (data[:confidence_values][i]!=nil and data[:confidence_values][i]>=min_confidence)) + else + valid = valid_indices.size<max_num_predictions + end + valid_indices << i if valid + end + [ :predicted_values, :actual_values, :confidence_values ].each do |key| + arr = [] + valid_indices.each{|i| arr << data[key][i]} + data[key] = arr + end + if compounds!=nil + new_compounds = [] + valid_indices.each{|i| new_compounds << compounds[i]} + end + LOGGER.debug("filtered predictions remaining: "+data[:predicted_values].size.to_s+"/"+orig_size.to_s) + + PredictionData.new(data, new_compounds) + end + + def data + @data + end + + def compounds + @compounds + end + + def self.create( feature_type, test_dataset_uris, test_target_dataset_uris, + prediction_feature, prediction_dataset_uris, predicted_variables, predicted_confidences, + subjectid=nil, task=nil ) + + test_dataset_uris = [test_dataset_uris] unless test_dataset_uris.is_a?(Array) + test_target_dataset_uris = [test_target_dataset_uris] unless test_target_dataset_uris.is_a?(Array) + prediction_dataset_uris = [prediction_dataset_uris] unless prediction_dataset_uris.is_a?(Array) + predicted_variables = [predicted_variables] unless predicted_variables.is_a?(Array) + predicted_confidences = [predicted_confidences] unless predicted_confidences.is_a?(Array) + LOGGER.debug "loading prediction -- test-dataset: "+test_dataset_uris.inspect + LOGGER.debug "loading prediction -- test-target-datset: "+test_target_dataset_uris.inspect + LOGGER.debug "loading prediction -- prediction-dataset: "+prediction_dataset_uris.inspect + LOGGER.debug "loading prediction -- predicted_variable: "+predicted_variables.inspect + LOGGER.debug "loading prediction -- predicted_confidence: "+predicted_confidences.inspect + LOGGER.debug "loading prediction -- prediction_feature: "+prediction_feature.to_s + raise "prediction_feature missing" unless prediction_feature + + all_compounds = [] + all_predicted_values = [] + all_actual_values = [] + all_confidence_values = [] + accept_values = nil + + if task + task_step = 100 / (test_dataset_uris.size*2 + 1) + task_status = 0 + end + + test_dataset_uris.size.times do |i| + + test_dataset_uri = test_dataset_uris[i] + test_target_dataset_uri = test_target_dataset_uris[i] + prediction_dataset_uri = prediction_dataset_uris[i] + predicted_variable = predicted_variables[i] + predicted_confidence = predicted_confidences[i] + + predicted_variable=prediction_feature if predicted_variable==nil + + test_dataset = Lib::DatasetCache.find test_dataset_uri,subjectid + raise "test dataset not found: '"+test_dataset_uri.to_s+"'" unless test_dataset + + if test_target_dataset_uri == nil || test_target_dataset_uri.strip.size==0 || test_target_dataset_uri==test_dataset_uri + test_target_dataset_uri = test_dataset_uri + test_target_dataset = test_dataset + raise "prediction_feature not found in test_dataset, specify a test_target_dataset\n"+ + "prediction_feature: '"+prediction_feature.to_s+"'\n"+ + "test_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ + "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil + else + test_target_dataset = Lib::DatasetCache.find test_target_dataset_uri,subjectid + raise "test target datset not found: '"+test_target_dataset_uri.to_s+"'" unless test_target_dataset + if CHECK_VALUES + test_dataset.compounds.each do |c| + raise "test compound not found on test class dataset "+c.to_s unless test_target_dataset.compounds.include?(c) + end + end + raise "prediction_feature not found in test_target_dataset\n"+ + "prediction_feature: '"+prediction_feature.to_s+"'\n"+ + "test_target_dataset: '"+test_target_dataset_uri.to_s+"'\n"+ + "available features are: "+test_target_dataset.features.inspect if test_target_dataset.features.keys.index(prediction_feature)==nil + end + + compounds = test_dataset.compounds + LOGGER.debug "test dataset size: "+compounds.size.to_s + raise "test dataset is empty "+test_dataset_uri.to_s unless compounds.size>0 + + if feature_type=="classification" + av = test_target_dataset.accept_values(prediction_feature) + raise "'"+OT.acceptValue.to_s+"' missing/invalid for feature '"+prediction_feature.to_s+"' in dataset '"+ + test_target_dataset_uri.to_s+"', acceptValues are: '"+av.inspect+"'" if av==nil or av.length<2 + if accept_values==nil + accept_values=av + else + raise "accept values (in folds) differ "+av.inspect+" != "+accept_values.inspect if av!=accept_values + end + end + + actual_values = [] + tmp_compounds = [] + compounds.each do |c| + case feature_type + when "classification" + vals = classification_vals(test_target_dataset, c, prediction_feature, accept_values) + when "regression" + vals = regression_vals(test_target_dataset, c, prediction_feature) + end + vals.each do |v| + actual_values << v + tmp_compounds << c + end + end + compounds = tmp_compounds + task.progress( task_status += task_step ) if task # loaded actual values + + prediction_dataset = Lib::DatasetCache.find prediction_dataset_uri,subjectid + raise "prediction dataset not found: '"+prediction_dataset_uri.to_s+"'" unless prediction_dataset + + # allow missing prediction feature if there are no compounds in the prediction dataset + raise "predicted_variable not found in prediction_dataset\n"+ + "predicted_variable '"+predicted_variable.to_s+"'\n"+ + "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+ + "available features are: "+prediction_dataset.features.inspect if prediction_dataset.features.keys.index(predicted_variable)==nil and prediction_dataset.compounds.size>0 + raise "predicted_confidence not found in prediction_dataset\n"+ + "predicted_confidence '"+predicted_confidence.to_s+"'\n"+ + "prediction_dataset: '"+prediction_dataset_uri.to_s+"'\n"+ + "available features are: "+prediction_dataset.features.inspect if predicted_confidence and prediction_dataset.features.keys.index(predicted_confidence)==nil and prediction_dataset.compounds.size>0 + + raise "more predicted than test compounds, #test: "+compounds.size.to_s+" < #prediction: "+ + prediction_dataset.compounds.size.to_s+", test-dataset: "+test_dataset_uri.to_s+", prediction-dataset: "+ + prediction_dataset_uri if compounds.size < prediction_dataset.compounds.size + if CHECK_VALUES + prediction_dataset.compounds.each do |c| + raise "predicted compound not found in test dataset:\n"+c+"\ntest-compounds:\n"+ + compounds.collect{|c| c.to_s}.join("\n") if compounds.index(c)==nil + end + end + + predicted_values = [] + confidence_values = [] + count = 0 + compounds.each do |c| + if prediction_dataset.compounds.index(c)==nil + predicted_values << nil + confidence_values << nil + else + case feature_type + when "classification" + vals = classification_vals(prediction_dataset, c, predicted_variable, accept_values) + when "regression" + vals = regression_vals(prediction_dataset, c, predicted_variable) + end + raise "not yet implemented: more than one prediction for one compound" if vals.size>1 + predicted_values << vals[0] + if predicted_confidence + confidence_values << confidence_val(prediction_dataset, c, predicted_confidence) + else + confidence_values << nil + end + end + count += 1 + end + all_compounds += compounds + all_predicted_values += predicted_values + all_actual_values += actual_values + all_confidence_values += confidence_values + + task.progress( task_status += task_step ) if task # loaded predicted values and confidence + end + + #sort according to confidence if available + if all_confidence_values.compact.size>0 + values = [] + all_predicted_values.size.times do |i| + values << [all_predicted_values[i], all_actual_values[i], all_confidence_values[i], all_compounds[i]] + end + values = values.sort_by{ |v| v[2] || 0 }.reverse # sorting by confidence + all_predicted_values = [] + all_actual_values = [] + all_confidence_values = [] + all_compounds = [] + values.each do |v| + all_predicted_values << v[0] + all_actual_values << v[1] + all_confidence_values << v[2] + all_compounds << v[3] + end + end + + raise "illegal num compounds "+all_compounds.size.to_s+" != "+all_predicted_values.size.to_s if + all_compounds.size != all_predicted_values.size + task.progress(100) if task # done with the mathmatics + data = { :predicted_values => all_predicted_values, :actual_values => all_actual_values, :confidence_values => all_confidence_values, + :feature_type => feature_type, :accept_values => accept_values } + + PredictionData.new(data, all_compounds) + end + + private + def initialize( data, compounds ) + @data = data + @compounds = compounds + end + + private + def self.regression_vals(dataset, compound, feature) + v_num = [] + values(dataset, compound, feature).each do |v| + if v==nil or v.is_a?(Numeric) + v_num << v + else + begin + v_num << v.to_f + rescue + LOGGER.warn "no numeric value for regression: '"+v.to_s+"'" + v_num << nil + end + end + end + v_num + end + + def self.confidence_val(dataset, compound, confidence) + v = values(dataset, compound, confidence) + raise "not yet implemented: duplicate conf value" if v.size>1 + begin + v = v[0] + v = v.to_f unless v==nil or v.is_a?(Numeric) + v + rescue + LOGGER.warn "no numeric value for confidence '"+v.to_s+"'" + nil + end + end + + def self.classification_vals(dataset, compound, feature, accept_values) + v_indices = [] + values(dataset, compound, feature).each do |v| + i = accept_values.index(v.to_s) + raise "illegal class_value of prediction (value is '"+v.to_s+"'), accept values are "+ + accept_values.inspect unless v==nil or i!=nil + v_indices << i + end + v_indices + end + + def self.values(dataset, compound, feature) + return [nil] if dataset.data_entries[compound]==nil + if feature==nil + v = dataset.data_entries[compound].values[0] + else + v = dataset.data_entries[compound][feature] + end + return [nil] if v==nil + # sanitiy checks + raise "no array "+v.class.to_s+" : '"+v.to_s+"'" unless v.is_a?(Array) + v.each{|vv| raise "array-elem is array" if vv.is_a?(Array)} + # replace empty strings with nil + v_mod = v.collect{|vv| (vv.to_s().size==0 ? nil : vv)} + v_mod + end + end +end
\ No newline at end of file diff --git a/lib/predictions.rb b/lib/predictions.rb index 6c0e996..233267d 100755 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -1,4 +1,6 @@ +require "lib/prediction_data.rb" + module Lib module Util @@ -18,18 +20,12 @@ module Lib def identifier(instance_index) return instance_index.to_s end - - def initialize( predicted_values, - actual_values, - confidence_values, - feature_type, - accept_values=nil ) + + def initialize( data ) + raise unless data.is_a?(Hash) - @predicted_values = predicted_values - @actual_values = actual_values - @confidence_values = confidence_values - @feature_type = feature_type - @accept_values = accept_values + @feature_type = data[:feature_type] + @accept_values = data[:accept_values] @num_classes = 1 #puts "predicted: "+predicted_values.inspect @@ -38,34 +34,27 @@ module Lib raise "unknown feature_type: '"+@feature_type.to_s+"'" unless @feature_type=="classification" || @feature_type=="regression" - raise "no predictions" if @predicted_values.size == 0 - num_info = "predicted:"+@predicted_values.size.to_s+ - " confidence:"+@confidence_values.size.to_s+" actual:"+@actual_values.size.to_s - raise "illegal num actual values "+num_info if @actual_values.size != @predicted_values.size - raise "illegal num confidence values "+num_info if @confidence_values.size != @predicted_values.size - - @confidence_values.each{ |c| raise "illegal confidence value: '"+c.to_s+"'" unless c==nil or (c.is_a?(Numeric) and c>=0 and c<=1) } + raise "no predictions" if data[:predicted_values].size == 0 + num_info = "predicted:"+data[:predicted_values].size.to_s+ + " confidence:"+data[:confidence_values].size.to_s+" actual:"+data[:actual_values].size.to_s + raise "illegal num actual values "+num_info if data[:actual_values].size != data[:predicted_values].size + raise "illegal num confidence values "+num_info if data[:confidence_values].size != data[:predicted_values].size case @feature_type when "classification" raise "accept_values missing while performing classification" unless @accept_values @num_classes = @accept_values.size raise "num classes < 2" if @num_classes<2 - { "predicted"=>@predicted_values, "actual"=>@actual_values }.each do |s,values| - values.each{ |v| raise "illegal "+s+" classification-value ("+v.to_s+"),"+ - "has to be either nil or index of predicted-values" if v!=nil and (!v.is_a?(Numeric) or v<0 or v>@num_classes)} - end when "regression" raise "accept_values != nil while performing regression" if @accept_values - { "predicted"=>@predicted_values, "actual"=>@actual_values }.each do |s,values| - values.each{ |v| raise "illegal "+s+" regression-value ("+v.to_s+"),"+ - " has to be either nil or number (not NaN, not Infinite)" unless v==nil or (v.is_a?(Numeric) and !v.nan? and v.finite?)} - end end + @predicted_values = [] + @actual_values = [] + @confidence_values = [] init_stats() - (0..@predicted_values.size-1).each do |i| - update_stats( @predicted_values[i], @actual_values[i], @confidence_values[i] ) + (0..data[:predicted_values].size-1).each do |i| + update_stats( data[:predicted_values][i], data[:actual_values][i], data[:confidence_values][i] ) end end @@ -114,6 +103,13 @@ module Lib @sum_squares_actual = 0 @sum_squares_predicted = 0 + @sum_confidence = 0 + @weighted_sum_actual = 0 + @weighted_sum_predicted = 0 + @weighted_sum_multiply = 0 + @weighted_sum_squares_actual = 0 + @weighted_sum_squares_predicted = 0 + @sum_weighted_abs_error = 0 @sum_weighted_squared_error = 0 end @@ -121,6 +117,25 @@ module Lib def update_stats( predicted_value, actual_value, confidence_value ) + raise "illegal confidence value: '"+confidence_value.to_s+"'" unless + confidence_value==nil or (confidence_value.is_a?(Numeric) and confidence_value>=0 and confidence_value<=1) + case @feature_type + when "classification" + { "predicted"=>predicted_value, "actual"=>actual_value }.each do |s,v| + raise "illegal "+s+" classification-value ("+v.to_s+"),"+ + "has to be either nil or index of predicted-values" if v!=nil and (!v.is_a?(Numeric) or v<0 or v>@num_classes) + end + when "regression" + { "predicted"=>predicted_value, "actual"=>actual_value }.each do |s,v| + raise "illegal "+s+" regression-value ("+v.to_s+"),"+ + " has to be either nil or number (not NaN, not Infinite)" unless v==nil or (v.is_a?(Numeric) and !v.nan? and v.finite?) + end + end + + @predicted_values << predicted_value + @actual_values << actual_value + @confidence_values << confidence_value + if actual_value==nil @num_no_actual_value += 1 else @@ -165,6 +180,16 @@ module Lib @sum_multiply += (actual_value*predicted_value) @sum_squares_actual += actual_value**2 @sum_squares_predicted += predicted_value**2 + + if @conf_provided + w_a = actual_value * confidence_value + w_p = predicted_value * confidence_value + @weighted_sum_actual += w_a + @weighted_sum_predicted += w_p + @weighted_sum_multiply += (w_a*w_p) if @conf_provided + @weighted_sum_squares_actual += w_a**2 if @conf_provided + @weighted_sum_squares_predicted += w_p**2 if @conf_provided + end end end end @@ -254,6 +279,15 @@ module Lib return res end + # returns acutal values for a certain prediction + def confusion_matrix_row(predicted_class_index) + r = [] + (0..@num_classes-1).each do |actual| + r << @confusion_matrix[actual][predicted_class_index] + end + return r + end + def area_under_roc(class_index=nil) return prediction_feature_value_map( lambda{ |i| area_under_roc(i) } ) if class_index==nil @@ -514,7 +548,7 @@ module Lib return @sum_squared_error end - def r_square + def r_square #_old #return sample_correlation_coefficient ** 2 # see http://en.wikipedia.org/wiki/Coefficient_of_determination#Definitions @@ -525,7 +559,7 @@ module Lib ( r_2.infinite? || r_2.nan? ) ? 0 : r_2 end - def weighted_r_square + def weighted_r_square #_old return 0 unless confidence_values_available? ss_tot = weighted_total_sum_of_squares return 0 if ss_tot==0 @@ -533,6 +567,16 @@ module Lib ( r_2.infinite? || r_2.nan? ) ? 0 : r_2 end + #def r_square + # # as implemted in R + # return sample_correlation_coefficient ** 2 + #end + + #def weighted_r_square + # # as implemted in R + # return weighted_sample_correlation_coefficient ** 2 + #end + def sample_correlation_coefficient begin # formula see http://en.wikipedia.org/wiki/Correlation_and_dependence#Pearson.27s_product-moment_coefficient @@ -543,6 +587,16 @@ module Lib rescue; 0; end end + def weighted_sample_correlation_coefficient + begin + # formula see http://en.wikipedia.org/wiki/Correlation_and_dependence#Pearson.27s_product-moment_coefficient + scc = ( @num_predicted * @weighted_sum_multiply - @weighted_sum_actual * @weighted_sum_predicted ) / + ( Math.sqrt( @num_predicted * @weighted_sum_squares_actual - @weighted_sum_actual**2 ) * + Math.sqrt( @num_predicted * @weighted_sum_squares_predicted - @weighted_sum_predicted**2 ) ) + ( scc.infinite? || scc.nan? ) ? 0 : scc + rescue; 0; end + end + def total_sum_of_squares #return @variance_actual * ( @num_predicted - 1 ) sum = 0 @@ -608,17 +662,23 @@ module Lib return h end - def get_prediction_values(actual_accept_value, predicted_accept_value) + def get_prediction_values(performance_attr, performance_accept_value) #puts "get_roc_values for class_value: "+class_value.to_s raise "no confidence values" unless confidence_values_available? #raise "no class-value specified" if class_value==nil + actual_accept_value = nil + predicted_accept_value = nil + if performance_attr==:true_positive_rate + actual_accept_value = performance_accept_value + elsif performance_attr==:positive_predictive_value + predicted_accept_value = performance_accept_value + end actual_class_index = @accept_values.index(actual_accept_value) if actual_accept_value!=nil raise "class not found '"+actual_accept_value.to_s+"' in "+@accept_values.inspect if (actual_accept_value!=nil && actual_class_index==nil) - predicted_class_index = @accept_values.index(predicted_accept_value) if predicted_accept_value!=nil - raise "class not found "+predicted_accept_value.to_s+" in "+@accept_values.inspect if (predicted_accept_value!=nil && predicted_class_index==nil) + raise "class not found '"+predicted_accept_value.to_s+"' in "+@accept_values.inspect if (predicted_accept_value!=nil && predicted_class_index==nil) c = []; p = []; a = [] (0..@predicted_values.size-1).each do |i| @@ -690,6 +750,10 @@ module Lib @conf_provided end + def min_confidence + @confidence_values[-1] + end + ################################################################################################################### #def compound(instance_index) @@ -697,6 +761,67 @@ module Lib #end private + def self.test_update + p=[0.4,0.2,0.3,0.5,0.8] + a=[0.45,0.21,0.25,0.55,0.75] + c = Array.new(p.size) + pred = Predictions.new(p,a,c,"regression") + puts pred.r_square + + pred = nil + p.size.times do |i| + if pred==nil + pred = Predictions.new([p[0]],[a[0]],[c[0]],"regression") + else + pred.update_stats(p[i],a[i],c[i]) + end + puts pred.r_square + end + end + + def self.test_r_square + require "rubygems" + require "opentox-ruby" + + max_deviation = rand * 0.9 + avg_deviation = max_deviation * 0.5 + + p = [] + a = [] + c = [] + (100 + rand(1000)).times do |i| + r = rand + deviation = rand * max_deviation + a << r + p << r + ((rand<0.5 ? -1 : 1) * deviation) + #c << 0.5 + if (deviation > avg_deviation) + c << 0.4 + else + c << 0.6 + end + #puts a[-1].to_s+" "+p[-1].to_s + end + puts "num values "+p.size.to_s + + pred = Predictions.new(p,a,c,"regression") + puts "internal" + #puts "r-square old "+pred.r_square_old.to_s + puts "cor "+pred.sample_correlation_coefficient.to_s + puts "weighted cor "+pred.weighted_sample_correlation_coefficient.to_s + puts "r-square "+pred.r_square.to_s + + puts "R" + @@r = RinRuby.new(true,false) unless defined?(@@r) and @@r + @@r.assign "v1",a + @@r.assign "v2",p + puts "r cor "+@@r.pull("cor(v1,v2)").to_s + @@r.eval "fit <- lm(v1 ~ v2)" + @@r.eval "sum <- summary(fit)" + puts "r r-square "+@@r.pull("sum$r.squared").to_s + puts "r adjusted-r-square "+@@r.pull("sum$adj.r.squared").to_s + end + def prediction_feature_value_map(proc) res = {} (0..@num_classes-1).each do |i| @@ -706,4 +831,12 @@ module Lib end end -end
\ No newline at end of file +end + +#class Float +# def to_s +# "%.5f" % self +# end +#end +##Lib::Predictions.test_update +#Lib::Predictions.test_r_square diff --git a/lib/validation_db.rb b/lib/validation_db.rb index be004fb..c3a3f71 100755 --- a/lib/validation_db.rb +++ b/lib/validation_db.rb @@ -72,6 +72,7 @@ module Validation attribute :classification_statistics_yaml attribute :regression_statistics_yaml attribute :finished + attribute :prediction_data_yaml index :model_uri index :validation_type @@ -99,6 +100,14 @@ module Validation def regression_statistics=(rs) self.regression_statistics_yaml = rs.to_yaml end + + def prediction_data + YAML.load(self.prediction_data_yaml) if self.prediction_data_yaml + end + + def prediction_data=(pd) + self.prediction_data_yaml = pd.to_yaml + end def save super |