update for grid search with features

3 files changed, 44 insertions, 53 deletions

diff --git a/nch/08_grid_search.rb b/nch/08_grid_search.rb
index c5ae2a3..993dc75 100755
--- a/nch/08_grid_search.rb
+++ b/nch/08_grid_search.rb
@@ -20,7 +20,7 @@ DATA.each do |d|
 #    new_feature_dataset_uri(d)=>"new-pc-features",
 #    ob_feature_dataset_uri(d)=>"ob-pc-features",
   }.each do |feature_dataset_uri,f_desc|
-    [ 0.65 ].each do |min_sim| # [0.2,0.4,0.6,0.8]
+    [ 0.2, 0.4, 0.6 ].each do |min_sim| # [0.2,0.4,0.6,0.8]
       [ 0.0 ].each do |min_train| #[0.0,0.1,0.2]
 
         puts "\nfeatures #{f_desc}"
diff --git a/nch/09_eval_grid.rb b/nch/09_eval_grid.rb
index 69bc31d..41db55f 100755
--- a/nch/09_eval_grid.rb
+++ b/nch/09_eval_grid.rb
@@ -6,68 +6,41 @@ require "bundler"
 Bundler.require
 
 res = YAML.load("---
-- :min_sim: 0.0
+- :min_sim: 0.65
   :min_train: 0.0
-  :cv: http://localhost:8087/validation/crossvalidation/54
-  :r_square: 0.45370176424121433
-  :unpredicted: '0'
-- :min_sim: 0.1
+  :cv: http://localhost:8087/validation/crossvalidation/73
+  :r_square: 0.28572954365573644
+  :unpredicted: '3'
+  :features: all-pc-features
+- :min_sim: 0.65
   :min_train: 0.0
-  :cv: http://localhost:8087/validation/crossvalidation/55
-  :r_square: 0.4595956327160562
-  :unpredicted: '0'
-- :min_sim: 0.2
+  :cv: http://localhost:8087/validation/crossvalidation/74
+  :r_square: 0.20115321576362188
+  :unpredicted: '3'
+  :features: new-pc-features
+- :min_sim: 0.65
   :min_train: 0.0
-  :cv: http://localhost:8087/validation/crossvalidation/56
-  :r_square: 0.4277448070809924
-  :unpredicted: '0'
-- :min_sim: 0.3
-  :min_train: 0.0
-  :cv: http://localhost:8087/validation/crossvalidation/57
-  :r_square: 0.44307217405529253
-  :unpredicted: '0'
-- :min_sim: 0.4
-  :min_train: 0.0
-  :cv: http://localhost:8087/validation/crossvalidation/58
-  :r_square: 0.4536563787494641
-  :unpredicted: '0'
-- :min_sim: 0.5
-  :min_train: 0.0
-  :cv: http://localhost:8087/validation/crossvalidation/59
-  :r_square: 0.4461298986550688
-  :unpredicted: '0'
-- :min_sim: 0.6
-  :min_train: 0.0
-  :cv: http://localhost:8087/validation/crossvalidation/60
-  :r_square: 0.4720263367371236
-  :unpredicted: '1'
-- :min_sim: 0.7
-  :min_train: 0.0
-  :cv: http://localhost:8087/validation/crossvalidation/61
-  :r_square: 0.5057497593073133
-  :unpredicted: '15'
-- :min_sim: 0.8
-  :min_train: 0.0
-  :cv: http://localhost:8087/validation/crossvalidation/62
-  :r_square: 0.5505636328364947
-  :unpredicted: '65'
-")
-
-all=182
+  :cv: http://localhost:8087/validation/crossvalidation/76
+  :r_square: 0.013357820165398548
+  :unpredicted: '5'
+  :features: ob-pc-features")
 
-min_train = [nil,0.1,0.2,0.3,0.4,0.5]
+min_train = [nil,0.1,0.2] #,0.3,0.4,0.5]
 
-s = [ "sim\\train" ]
+all = nil
+s = [ "feat\\train" ]
 min_train.each do |m|
   s << (m==nil ? 0.0 : m).to_s
 end
 out = [s]
 
 res.each do |r|
-  s = [ r[:min_sim].to_s ]
+#  s = [ r[:min_sim].to_s ]
+ s = [ r[:features].to_s ]
   stats = OpenTox::Crossvalidation.find(r[:cv]).statistics
+  all = stats.metadata[RDF::OT.numInstances.to_s].to_i unless all
   min_train.each do |m|
-    data = (m==nil ? stats.metadata : stats.filter(m))
+    data = (m==nil ? stats.metadata : stats.filter_metadata(m))
     v =  sprintf("%.3f",data[RDF::OT.regressionStatistics.to_s][RDF::OT.rSquare.to_s])
     v << "("
     v << sprintf("%2d",(all - (data[RDF::OT.numInstances.to_s].to_i-data[RDF::OT.numUnpredicted.to_s].to_i)))
@@ -78,7 +51,7 @@ res.each do |r|
 end
 
 
-def print_2d_array(a, cs=10)
+def print_2d_array(a, cs=15)
   report = []   
   report << a.enum_for(:each_with_index).map { |ia, i|
     ia.map{|e| "%#{cs}s" % e}.join(" | ") }
diff --git a/nch/results/grid.yaml b/nch/results/grid.yaml
index fb136a7..efe99d8 100644
--- a/nch/results/grid.yaml
+++ b/nch/results/grid.yaml
@@ -126,8 +126,9 @@ min_sim 0
   :unpredicted: '44'
 
 
-skip_ratio = 0.8
-random_seed = 3
+# comparin features
+# skip_ratio = 0.8
+# random_seed = 3
 ---
 - :min_sim: 0.65
   :min_train: 0.0
@@ -149,3 +150,20 @@ random_seed = 3
   :features: ob-pc-features
 
 
+# all features on complete loocv
+---
+- :min_sim: 0.65
+  :min_train: 0.0
+  :cv: http://localhost:8087/validation/crossvalidation/77
+  :r_square: 0.4914116499931924
+  :unpredicted: '29'
+  :features: all-pc-features
+
+# compare different min_sims with all-features on complete loocv
+---
+- :min_sim: 0.2
+  :min_train: 0.0
+  :cv: http://localhost:8087/validation/crossvalidation/78
+  :r_square: 0.5033017681500125
+  :unpredicted: '0'
+  :features: all-pc-features