Added some validation tool for pc ds

3 files changed, 115 insertions, 0 deletions

diff --git a/5x_cv/CV_ds_pctype_prop_algo_rseed.rb b/5x_cv/CV_ds_pctype_prop_algo_rseed.rb
new file mode 100644
index 0000000..b9c55dd
--- /dev/null
+++ b/5x_cv/CV_ds_pctype_prop_algo_rseed.rb
@@ -0,0 +1,60 @@
+# Do a 10-fold crossvalidation
+# # Author: Andreas Maunz, David Vorgrimmler
+# # @params: Dataset_name(see dataset_nestle.yaml), pc_type(electronic,cpsa or constitutional ...), prop(true or false), prediction_algorithm(local_mlr_prop or local_svm_regression ...)
+
+if ARGV.size != 5 
+  puts "Args: ds_name, pc_type, prop, algo, random_seed"
+  puts ARGV.size
+  exit
+end
+
+ds_file = "datasets_nestle.yaml"
+pwd=`pwd`
+path = "#{pwd.chop}/#{ds_file}"
+if File.exists?(path)
+  puts "#{ds_file} exists"
+else
+  puts "#{ds_file} does not exist."
+  exit
+end
+
+require 'rubygems'
+require 'opentox-ruby'
+require 'yaml'
+
+subjectid = nil
+
+ds_name = ARGV[0] # e.g. MOU
+pc_type = ARGV[1] # e.g. electronic,cpsa
+prop = ARGV[2]    # true or false
+algo = ARGV[3]    # e.g. local_svm_regression, local_mlr_prop
+r_seed = ARGV[4]  # 1, 2, ..., 10
+
+ds = YAML::load_file("datasets_nestle.yaml")
+ds_uri = ds[ds_name]["dataset"]
+pc_ds_uri = ds[ds_name][pc_type]
+
+algo_params = "pc_type=#{pc_type}";
+algo_params += ";feature_dataset_uri=#{pc_ds_uri}"
+algo_params += ";propositionalized=#{prop}"
+algo_params += ";prediction_algorithm=#{algo}"
+puts algo_params.to_yaml
+
+prediction_feature = OpenTox::Dataset.find(ds_uri).features.keys.first
+
+
+# Ready
+cv_args = {}
+cv_args[:dataset_uri] = ds_uri
+cv_args[:prediction_feature] = prediction_feature
+cv_args[:algorithm_uri] = "http://toxcreate3.in-silico.ch:8087/algorithm/lazar"
+cv_args[:algorithm_params] = algo_params
+cv_args[:stratified] = false
+cv_args[:random_seed] = r_seed
+puts cv_args.to_yaml
+
+cv = OpenTox::Crossvalidation.create(cv_args).uri
+puts cv
+
+cvr = OpenTox::CrossvalidationReport.create( cv , subjectid).uri
+puts cvr
diff --git a/5x_cv/factors_config b/5x_cv/factors_config
new file mode 100644
index 0000000..4e3f4c5
--- /dev/null
+++ b/5x_cv/factors_config
@@ -0,0 +1,10 @@
+#Dataset pc_type prop prediction_algorithm
+MOU constitutional false local_svm_regression
+MOU constitutional true local_svm_regression
+MOU electronic,cpsa false local_svm_regression
+MOU electronic,cpsa true local_svm_regression
+MOU constitutional,electronic,cpsa false local_svm_regression
+MOU constitutional,electronic,cpsa true local_svm_regression
+#MOU constitutional true local_mlr_prop
+#MOU electronic,cpsa true local_mlr_prop
+#MOU constitutional,electronic,cpsa true local_mlr_prop
diff --git a/5x_cv/wrapper_pc_cv.sh b/5x_cv/wrapper_pc_cv.sh
new file mode 100755
index 0000000..719885a
--- /dev/null
+++ b/5x_cv/wrapper_pc_cv.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Wrapper Skript for CV
+# Reads factors_config, dataset_nestle.yaml and performs cv's
+# Andreas Maunz, David Vorgrimmler,  2012
+
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 factors"
+  exit
+fi
+
+PWD=`pwd`
+echo $PWD
+if [ ! -f $PWD/datasets_nestle.yaml ] 
+then
+  echo "datasets_nestle.yaml does not exist."
+  exit
+fi
+
+# Configure basics
+source $HOME/.bash_aliases
+otconfig
+THIS_DATE=`date +%Y%m%d_%H_`
+CV="CV_ds_pctype_prop_algo_rseed.rb"
+FACTORS="$1"
+
+# Don't start when running
+while ps x | grep $CV | grep -v grep >/dev/null 2>&1; do sleep 3; done
+
+LOGFILE="$THIS_DATE""$USER""_wrapper_pc_cv.log"
+rm "$LOGFILE" >/dev/null 2>&1
+
+
+cat $FACTORS | while read factor; do
+  if ! [[ "$factor" =~ "#" ]]; then # allow comments
+    for r_seed in 1 #2 3 4 5
+    do
+      factor="$factor $r_seed"
+      echo "${THIS_DATE}: $factor" >> $LOGFILE>&1
+      echo "ruby $CV $factor" >> $LOGFILE 2>&1
+      ruby $CV $factor >> $LOGFILE 2>&1
+      echo >> $LOGFILE 2>&1
+    done
+  fi
+done
+