summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormguetlein <martin.guetlein@gmail.com>2010-05-31 17:25:21 +0200
committermguetlein <martin.guetlein@gmail.com>2010-05-31 17:25:21 +0200
commit7ce7b16b6b0d2e965e92ae28e6f58644203c1cab (patch)
treea31f523060c15fb6a9525b010959eda3330f0b07
parentd09a2f15a7dc499545732d7b8b45601533252ed9 (diff)
test/nightly rewrite
Diffstat
-rw-r--r--application.rb1
-rw-r--r--data/hamster_carcinogenicity.test.yaml119
-rw-r--r--data/hamster_carcinogenicity.train.yaml178
-rw-r--r--nightly/nightly.rb598
-rw-r--r--nightly/nightly_application.rb4
-rw-r--r--report/report_test.rb10
-rw-r--r--test/test_application.rb17
-rw-r--r--test/test_examples.rb214
-rw-r--r--test/test_examples_util.rb194
-rw-r--r--validation/validation_test.rb631
10 files changed, 831 insertions, 1135 deletions
diff --git a/application.rb b/application.rb
index ca2a7ac..236280c 100644
--- a/application.rb
+++ b/application.rb
@@ -29,6 +29,7 @@ get '/test_examples/?' do
Example.test_examples
end
+require "test/test_application.rb"
require "nightly/nightly_application.rb"
# order is important, first add example methods and reports, than validation
diff --git a/data/hamster_carcinogenicity.test.yaml b/data/hamster_carcinogenicity.test.yaml
new file mode 100644
index 0000000..32d305a
--- /dev/null
+++ b/data/hamster_carcinogenicity.test.yaml
@@ -0,0 +1,119 @@
+--- !ruby/object:OpenTox::Dataset
+compounds:
+- http://localhost/compound/InChI=1S/C9H6O2/c10-9-6-5-7-3-1-2-4-8(7)11-9/h1-6H
+- http://localhost/compound/InChI=1S/2C2H4O2.4H2O.3Pb/c2*1-2(3)4;;;;;;;/h2*1H3,(H,3,4);4*1H2;;;/q;;;;;;3*+2/p-6
+- http://localhost/compound/InChI=1S/C10H12ClNO2/c1-7(2)14-10(13)12-9-5-3-4-8(11)6-9/h3-7H,1-2H3,(H,12,13)
+- http://localhost/compound/InChI=1S/C8H6N4O4S/c13-4-9-11-8-10-5(3-17-8)6-1-2-7(16-6)12(14)15/h1-4H,(H,9,13)(H,10,11)
+- http://localhost/compound/InChI=1S/C4H7N3O3/c1-3(8)2-7(6-10)4(5)9/h2H2,1H3,(H2,5,9)
+- http://localhost/compound/InChI=1S/C5H10N2O/c8-6-7-4-2-1-3-5-7/h1-5H2
+- http://localhost/compound/InChI=1S/C15H13NO/c1-10(17)16-13-6-7-15-12(9-13)8-11-4-2-3-5-14(11)15/h2-7,9H,8H2,1H3,(H,16,17)
+- http://localhost/compound/InChI=1S/C4H6N2O3/c1-3-2-6(5-8)4(7)9-3/h3H,2H2,1H3
+- http://localhost/compound/InChI=1S/CH6N2/c1-3-2/h3H,2H2,1H3
+- http://localhost/compound/InChI=1S/C2H6N2O/c1-4(3)2-5/h2H,3H2,1H3
+- http://localhost/compound/InChI=1S/C6H12N2O2/c1-5-3-8(7-9)4-6(2)10-5/h5-6H,3-4H2,1-2H3
+- http://localhost/compound/InChI=1S/C9H11N3O2/c10-9(13)12(11-14)7-6-8-4-2-1-3-5-8/h1-5H,6-7H2,(H2,10,13)
+- http://localhost/compound/InChI=1S/C5H6N2OS/c1-3-2-4(8)7-5(9)6-3/h2H,1H3,(H2,6,7,8,9)
+- http://localhost/compound/InChI=1S/C5H11N3O3/c1-2-8(7-11)5(10)6-3-4-9/h9H,2-4H2,1H3,(H,6,10)
+- http://localhost/compound/InChI=1S/C3H7NO2/c1-2-6-3(4)5/h2H2,1H3,(H2,4,5)
+- http://localhost/compound/InChI=1S/C14H19N3S.ClH/c1-16(2)9-10-17(12-13-6-5-11-18-13)14-7-3-4-8-15-14;/h3-8,11H,9-10,12H2,1-2H3;1H
+- http://localhost/compound/InChI=1S/C5H4O2/c6-4-5-2-1-3-7-5/h1-4H
+- http://localhost/compound/InChI=1S/C14H9Cl5/c15-11-5-1-9(2-6-11)13(14(17,18)19)10-3-7-12(16)8-4-10/h1-8,13H
+- http://localhost/compound/InChI=1S/H4N2.H2O4S/c1-2;1-5(2,3)4/h1-2H2;(H2,1,2,3,4)
+- http://localhost/compound/InChI=1S/C4H8N2O3/c1-3-9-4(7)6(2)5-8/h3H2,1-2H3
+- http://localhost/compound/InChI=1S/C3H6O2/c4-1-3-2-5-3/h3-4H,1-2H2
+- http://localhost/compound/InChI=1S/C19H17N3.ClH/c20-16-7-1-13(2-8-16)19(14-3-9-17(21)10-4-14)15-5-11-18(22)12-6-15;/h1-12,20H,21-22H2;1H
+- http://localhost/compound/InChI=1S/C9H7N3O4S/c1-5(13)10-9-11-6(4-17-9)7-2-3-8(16-7)12(14)15/h2-4H,1H3,(H,10,11,13)
+- http://localhost/compound/InChI=1S/C6H12N4O2/c1-5-3-9(7-11)4-6(2)10(5)8-12/h5-6H,3-4H2,1-2H3
+- http://localhost/compound/InChI=1S/C15H10O7.2H2O/c16-7-4-10(19)12-11(5-7)22-15(14(21)13(12)20)6-1-2-8(17)9(18)3-6;;/h1-5,16-19,21H;2*1H2
+- http://localhost/compound/InChI=1S/C6H7N3O/c7-9-6(10)5-1-3-8-4-2-5/h1-4H,7H2,(H,9,10)
+- http://localhost/compound/InChI=1S/C6H11N3O3/c1-3-9(8-12)6(11)7-4-5(2)10/h3-4H2,1-2H3,(H,7,11)
+- http://localhost/compound/InChI=1S/C5H8O2/c1-4(2)5(6)7-3/h1H2,2-3H3
+creator: http://example.org/training_test_split
+data:
+ http://localhost/compound/InChI=1S/C9H11N3O2/c10-9(13)12(11-14)7-6-8-4-2-1-3-5-8/h1-5H,6-7H2,(H2,10,13):
+ - {}
+
+ http://localhost/compound/InChI=1S/C9H6O2/c10-9-6-5-7-3-1-2-4-8(7)11-9/h1-6H:
+ - {}
+
+ http://localhost/compound/InChI=1S/C3H6O2/c4-1-3-2-5-3/h3-4H,1-2H2:
+ - {}
+
+ http://localhost/compound/InChI=1S/C4H8N2O3/c1-3-9-4(7)6(2)5-8/h3H2,1-2H3:
+ - {}
+
+ http://localhost/compound/InChI=1S/C5H6N2OS/c1-3-2-4(8)7-5(9)6-3/h2H,1H3,(H2,6,7,8,9):
+ - {}
+
+ http://localhost/compound/InChI=1S/C6H12N2O2/c1-5-3-8(7-9)4-6(2)10-5/h5-6H,3-4H2,1-2H3:
+ - {}
+
+ http://localhost/compound/InChI=1S/C10H12ClNO2/c1-7(2)14-10(13)12-9-5-3-4-8(11)6-9/h3-7H,1-2H3,(H,12,13):
+ - {}
+
+ http://localhost/compound/InChI=1S/C9H7N3O4S/c1-5(13)10-9-11-6(4-17-9)7-2-3-8(16-7)12(14)15/h2-4H,1H3,(H,10,11,13):
+ - {}
+
+ http://localhost/compound/InChI=1S/C6H11N3O3/c1-3-9(8-12)6(11)7-4-5(2)10/h3-4H2,1-2H3,(H,7,11):
+ - {}
+
+ http://localhost/compound/InChI=1S/C5H4O2/c6-4-5-2-1-3-7-5/h1-4H:
+ - {}
+
+ http://localhost/compound/InChI=1S/C8H6N4O4S/c13-4-9-11-8-10-5(3-17-8)6-1-2-7(16-6)12(14)15/h1-4H,(H,9,13)(H,10,11):
+ - {}
+
+ http://localhost/compound/InChI=1S/C14H9Cl5/c15-11-5-1-9(2-6-11)13(14(17,18)19)10-3-7-12(16)8-4-10/h1-8,13H:
+ - {}
+
+ http://localhost/compound/InChI=1S/C3H7NO2/c1-2-6-3(4)5/h2H2,1H3,(H2,4,5):
+ - {}
+
+ http://localhost/compound/InChI=1S/C15H13NO/c1-10(17)16-13-6-7-15-12(9-13)8-11-4-2-3-5-14(11)15/h2-7,9H,8H2,1H3,(H,16,17):
+ - {}
+
+ http://localhost/compound/InChI=1S/C5H10N2O/c8-6-7-4-2-1-3-5-7/h1-5H2:
+ - {}
+
+ http://localhost/compound/InChI=1S/C19H17N3.ClH/c20-16-7-1-13(2-8-16)19(14-3-9-17(21)10-4-14)15-5-11-18(22)12-6-15;/h1-12,20H,21-22H2;1H:
+ - {}
+
+ http://localhost/compound/InChI=1S/H4N2.H2O4S/c1-2;1-5(2,3)4/h1-2H2;(H2,1,2,3,4):
+ - {}
+
+ http://localhost/compound/InChI=1S/C6H7N3O/c7-9-6(10)5-1-3-8-4-2-5/h1-4H,7H2,(H,9,10):
+ - {}
+
+ http://localhost/compound/InChI=1S/CH6N2/c1-3-2/h3H,2H2,1H3:
+ - {}
+
+ http://localhost/compound/InChI=1S/C6H12N4O2/c1-5-3-9(7-11)4-6(2)10(5)8-12/h5-6H,3-4H2,1-2H3:
+ - {}
+
+ http://localhost/compound/InChI=1S/C2H6N2O/c1-4(3)2-5/h2H,3H2,1H3:
+ - {}
+
+ http://localhost/compound/InChI=1S/C4H6N2O3/c1-3-2-6(5-8)4(7)9-3/h3H,2H2,1H3:
+ - {}
+
+ http://localhost/compound/InChI=1S/C14H19N3S.ClH/c1-16(2)9-10-17(12-13-6-5-11-18-13)14-7-3-4-8-15-14;/h3-8,11H,9-10,12H2,1-2H3;1H:
+ - {}
+
+ http://localhost/compound/InChI=1S/C4H7N3O3/c1-3(8)2-7(6-10)4(5)9/h2H2,1H3,(H2,5,9):
+ - {}
+
+ http://localhost/compound/InChI=1S/2C2H4O2.4H2O.3Pb/c2*1-2(3)4;;;;;;;/h2*1H3,(H,3,4);4*1H2;;;/q;;;;;;3*+2/p-6:
+ - {}
+
+ http://localhost/compound/InChI=1S/C5H8O2/c1-4(2)5(6)7-3/h1H2,2-3H3:
+ - {}
+
+ http://localhost/compound/InChI=1S/C15H10O7.2H2O/c16-7-4-10(19)12-11(5-7)22-15(14(21)13(12)20)6-1-2-8(17)9(18)3-6;;/h1-5,16-19,21H;2*1H2:
+ - {}
+
+ http://localhost/compound/InChI=1S/C5H11N3O3/c1-2-8(7-11)5(10)6-3-4-9/h9H,2-4H2,1H3,(H,6,10):
+ - {}
+
+features: []
+
+title: Test dataset split of Hamster Carcinogenicity (DSSTOX/CPDB)
diff --git a/data/hamster_carcinogenicity.train.yaml b/data/hamster_carcinogenicity.train.yaml
new file mode 100644
index 0000000..fd1544f
--- /dev/null
+++ b/data/hamster_carcinogenicity.train.yaml
@@ -0,0 +1,178 @@
+--- !ruby/object:OpenTox::Dataset
+compounds:
+- http://localhost/compound/InChI=1S/C11H8N2O5/c12-11(14)8(9-2-1-5-17-9)6-7-3-4-10(18-7)13(15)16/h1-6H,(H2,12,14)
+- http://localhost/compound/InChI=1S/C20H22N8O5/c1-28(9-11-8-23-17-15(24-11)16(21)26-20(22)27-17)12-4-2-10(3-5-12)18(31)25-13(19(32)33)6-7-14(29)30/h2-5,8,13H,6-7,9H2,1H3,(H,25,31)(H,29,30)(H,32,33)(H4,21,22,23,26,27)/t13-/m0/s1
+- http://localhost/compound/InChI=1S/C6H12N2O4/c1-5(10)2-8(7-12)3-6(11)4-9/h6,9,11H,2-4H2,1H3
+- http://localhost/compound/InChI=1S/C6H10N2O/c1-3-5-8(7-9)6-4-2/h3-4H,1-2,5-6H2
+- http://localhost/compound/InChI=1S/C6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9
+- http://localhost/compound/InChI=1S/C12H12N2O3/c1-2-12(8-6-4-3-5-7-8)9(15)13-11(17)14-10(12)16/h3-7H,2H2,1H3,(H2,13,14,15,16,17)
+- http://localhost/compound/InChI=1S/C4H10N2O3/c1-6(5-9)2-4(8)3-7/h4,7-8H,2-3H2,1H3
+- http://localhost/compound/InChI=1S/C17H17ClO6/c1-8-5-9(19)6-12(23-4)17(8)16(20)13-10(21-2)7-11(22-3)14(18)15(13)24-17/h6-8H,5H2,1-4H3/t8-,17?/m1/s1
+- http://localhost/compound/InChI=1S/Cd.2ClH/h;2*1H/q+2;;/p-2
+- http://localhost/compound/InChI=1S/C15H13NO2/c1-10(17)16(18)13-6-7-15-12(9-13)8-11-4-2-3-5-14(11)15/h2-7,9,18H,8H2,1H3
+- http://localhost/compound/InChI=1S/C4H8N2O/c7-5-6-3-1-2-4-6/h1-4H2
+- http://localhost/compound/InChI=1S/C20H22O3/c1-20(2,19(21)22)23-16-12-10-15(11-13-16)18-9-5-7-14-6-3-4-8-17(14)18/h3-4,6,8,10-13,18H,5,7,9H2,1-2H3,(H,21,22)
+- http://localhost/compound/InChI=1S/C6H14N2O4/c1-5(10)2-8(7-12)3-6(11)4-9/h5-6,9-11H,2-4H2,1H3
+- http://localhost/compound/InChI=1S/C27H30O16/c1-8-17(32)20(35)22(37)26(40-8)39-7-15-18(33)21(36)23(38)27(42-15)43-25-19(34)16-13(31)5-10(28)6-14(16)41-24(25)9-2-3-11(29)12(30)4-9/h2-6,8,15,17-18,20-23,26-33,35-38H,7H2,1H3/t8-,15+,17-,18+,20+,21-,22+,23+,26+,27?/m0/s1
+- http://localhost/compound/InChI=1S/C2HCl3/c3-1-2(4)5/h1H
+- http://localhost/compound/InChI=1S/C2H8N2.2ClH/c1-3-4-2;;/h3-4H,1-2H3;2*1H
+- http://localhost/compound/InChI=1S/C9H11N3O/c13-11-12-6-2-4-9(12)8-3-1-5-10-7-8/h1,3,5,7,9H,2,4,6H2
+- http://localhost/compound/InChI=1S/BrHO3.K/c2-1(3)4;/h(H,2,3,4);/q;+1/p-1
+- http://localhost/compound/InChI=1S/C6H5N2.BF4/c7-8-6-4-2-1-3-5-6;2-1(3,4)5/h1-5H;/q+1;-1
+- http://localhost/compound/InChI=1S/H4N2/c1-2/h1-2H2
+- http://localhost/compound/InChI=1S/Cd.H2O4S/c;1-5(2,3)4/h;(H2,1,2,3,4)/q+2;/p-2
+- http://localhost/compound/InChI=1S/C2H6O/c1-2-3/h3H,2H2,1H3
+- http://localhost/compound/InChI=1S/C14H8Cl4/c15-11-5-1-9(2-6-11)13(14(17)18)10-3-7-12(16)8-4-10/h1-8H
+- http://localhost/compound/InChI=1S/C5H10N2O3/c1-5(9)4-7(6-10)2-3-8/h8H,2-4H2,1H3
+- http://localhost/compound/InChI=1S/C7H15N3O/c1-6-4-10(8-11)5-7(2)9(6)3/h6-7H,4-5H2,1-3H3
+- http://localhost/compound/InChI=1S/BF4.Na/c2-1(3,4)5;/q-1;+1
+- http://localhost/compound/InChI=1S/C10H13N3O2/c1-13(12-15)7-3-5-10(14)9-4-2-6-11-8-9/h2,4,6,8H,3,5,7H2,1H3
+- http://localhost/compound/InChI=1S/C12H9NO2/c14-13(15)11-7-6-9-5-4-8-2-1-3-10(11)12(8)9/h1-3,6-7H,4-5H2
+- http://localhost/compound/InChI=1S/C8H5N3O4S/c12-4-9-8-10-5(3-16-8)6-1-2-7(15-6)11(13)14/h1-4H,(H,9,10,12)
+- http://localhost/compound/InChI=1S/C2H4N4/c3-2-4-1-5-6-2/h1H,(H3,3,4,5,6)
+- http://localhost/compound/InChI=1S/CH2O/c1-2/h1H2
+- http://localhost/compound/InChI=1S/C10H13NO2/c1-8(2)13-10(12)11-9-6-4-3-5-7-9/h3-8H,1-2H3,(H,11,12)
+- http://localhost/compound/InChI=1S/C14H14ClN3O2S/c1-8-4-3-5-10(9(8)2)16-12-6-11(15)17-14(18-12)21-7-13(19)20/h3-6H,7H2,1-2H3,(H,19,20)(H,16,17,18)
+- http://localhost/compound/InChI=1S/C5H13N3O/c1-7(2)4-5-8(3)6-9/h4-5H2,1-3H3
+- http://localhost/compound/InChI=1S/C3H6N2O/c6-4-5-2-1-3-5/h1-3H2
+- http://localhost/compound/InChI=1S/C2H5ClO/c1-4-2-3/h2H2,1H3
+- http://localhost/compound/InChI=1S/C2H3Cl/c1-2-3/h2H,1H2
+- http://localhost/compound/InChI=1S/C6H10ClN3O3/c1-5(11)4-10(9-13)6(12)8-3-2-7/h2-4H2,1H3,(H,8,12)
+- http://localhost/compound/InChI=1S/C9H9NS/c11-8-10-7-6-9-4-2-1-3-5-9/h1-5H,6-7H2
+- http://localhost/compound/InChI=1S/C2H4O/c1-2-3/h2H,1H3
+- http://localhost/compound/InChI=1S/C3H6ClNO/c1-5(2)3(4)6/h1-2H3
+- http://localhost/compound/InChI=1S/C17H17ClO3/c1-17(2,16(19)20)21-11-12-3-5-13(6-4-12)14-7-9-15(18)10-8-14/h3-10H,11H2,1-2H3,(H,19,20)
+- http://localhost/compound/InChI=1S/C2H8N2O/c3-4-1-2-5/h4-5H,1-3H2
+- http://localhost/compound/InChI=1S/HNO2.Na/c2-1-3;/h(H,2,3);/q;+1/p-1
+- http://localhost/compound/InChI=1S/C6H10N2O2/c1-3-4-8(7-10)5-6(2)9/h3H,1,4-5H2,2H3
+- http://localhost/compound/InChI=1S/C7H6O4/c8-5-2-1-4(7(10)11)3-6(5)9/h1-3,8-9H,(H,10,11)
+- http://localhost/compound/InChI=1S/C6H5NO2/c8-6(9)5-1-3-7-4-2-5/h1-4H,(H,8,9)
+- http://localhost/compound/InChI=1S/C16H13N/c1-2-8-15(9-3-1)17-16-11-10-13-6-4-5-7-14(13)12-16/h1-12,17H
+- http://localhost/compound/InChI=1S/C4H5Cl/c1-3-4(2)5/h3H,1-2H2
+- http://localhost/compound/InChI=1S/C2H8N2/c1-4(2)3/h3H2,1-2H3
+- http://localhost/compound/InChI=1S/C4H8N2O2/c7-5-6-1-3-8-4-2-6/h1-4H2
+- http://localhost/compound/InChI=1S/C20H19N3.ClH/c1-13-12-16(6-11-19(13)23)20(14-2-7-17(21)8-3-14)15-4-9-18(22)10-5-15;/h2-12,21H,22-23H2,1H3;1H
+- http://localhost/compound/InChI=1S/C9H11N3O2/c13-10-12-6-2-4-9(12)8-3-1-5-11(14)7-8/h1,3,5,7,9H,2,4,6H2
+- http://localhost/compound/InChI=1S/C3H6N2O2/c6-4-5-1-2-7-3-5/h1-3H2
+- http://localhost/compound/InChI=1S/C19H24N2O2/c22-18-13-20(19(23)15-7-2-1-3-8-15)12-17-16-9-5-4-6-14(16)10-11-21(17)18/h4-6,9,15,17H,1-3,7-8,10-13H2
+- http://localhost/compound/InChI=1S/C12H8Cl6O/c13-8-9(14)11(16)5-3-1-2(6-7(3)19-6)4(5)10(8,15)12(11,17)18/h2-7H,1H2
+- http://localhost/compound/InChI=1S/C5H12N2O4/c8-2-1-7(6-11)3-5(10)4-9/h5,8-10H,1-4H2
+creator: http://example.org/training_test_split
+data:
+ http://localhost/compound/InChI=1S/C7H6O4/c8-5-2-1-4(7(10)11)3-6(5)9/h1-3,8-9H,(H,10,11):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C3H6N2O/c6-4-5-2-1-3-5/h1-3H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C10H13N3O2/c1-13(12-15)7-3-5-10(14)9-4-2-6-11-8-9/h2,4,6,8H,3,5,7H2,1H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C2H6O/c1-2-3/h3H,2H2,1H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C6H14N2O4/c1-5(10)2-8(7-12)3-6(11)4-9/h5-6,9-11H,2-4H2,1H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C4H8N2O/c7-5-6-3-1-2-4-6/h1-4H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/Cd.2ClH/h;2*1H/q+2;;/p-2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C17H17ClO6/c1-8-5-9(19)6-12(23-4)17(8)16(20)13-10(21-2)7-11(22-3)14(18)15(13)24-17/h6-8H,5H2,1-4H3/t8-,17?/m1/s1:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C4H10N2O3/c1-6(5-9)2-4(8)3-7/h4,7-8H,2-3H2,1H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C20H22N8O5/c1-28(9-11-8-23-17-15(24-11)16(21)26-20(22)27-17)12-4-2-10(3-5-12)18(31)25-13(19(32)33)6-7-14(29)30/h2-5,8,13H,6-7,9H2,1H3,(H,25,31)(H,29,30)(H,32,33)(H4,21,22,23,26,27)/t13-/m0/s1:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C2H8N2O/c3-4-1-2-5/h4-5H,1-3H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C17H17ClO3/c1-17(2,16(19)20)21-11-12-3-5-13(6-4-12)14-7-9-15(18)10-8-14/h3-10H,11H2,1-2H3,(H,19,20):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C2H4O/c1-2-3/h2H,1H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C2H5ClO/c1-4-2-3/h2H2,1H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C8H5N3O4S/c12-4-9-8-10-5(3-16-8)6-1-2-7(15-6)11(13)14/h1-4H,(H,9,10,12):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C9H11N3O/c13-11-12-6-2-4-9(12)8-3-1-5-10-7-8/h1,3,5,7,9H,2,4,6H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C20H22O3/c1-20(2,19(21)22)23-16-12-10-15(11-13-16)18-9-5-7-14-6-3-4-8-17(14)18/h3-4,6,8,10-13,18H,5,7,9H2,1-2H3,(H,21,22):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C11H8N2O5/c12-11(14)8(9-2-1-5-17-9)6-7-3-4-10(18-7)13(15)16/h1-6H,(H2,12,14):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C20H19N3.ClH/c1-13-12-16(6-11-19(13)23)20(14-2-7-17(21)8-3-14)15-4-9-18(22)10-5-15;/h2-12,21H,22-23H2,1H3;1H:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C16H13N/c1-2-8-15(9-3-1)17-16-11-10-13-6-4-5-7-14(13)12-16/h1-12,17H:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C12H9NO2/c14-13(15)11-7-6-9-5-4-8-2-1-3-10(11)12(8)9/h1-3,6-7H,4-5H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C15H13NO2/c1-10(17)16(18)13-6-7-15-12(9-13)8-11-4-2-3-5-14(11)15/h2-7,9,18H,8H2,1H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C4H8N2O2/c7-5-6-1-3-8-4-2-6/h1-4H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C9H9NS/c11-8-10-7-6-9-4-2-1-3-5-9/h1-5H,6-7H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/BF4.Na/c2-1(3,4)5;/q-1;+1:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C7H15N3O/c1-6-4-10(8-11)5-7(2)9(6)3/h6-7H,4-5H2,1-3H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C5H10N2O3/c1-5(9)4-7(6-10)2-3-8/h8H,2-4H2,1H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/Cd.H2O4S/c;1-5(2,3)4/h;(H2,1,2,3,4)/q+2;/p-2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/BrHO3.K/c2-1(3)4;/h(H,2,3,4);/q;+1/p-1:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C3H6N2O2/c6-4-5-1-2-7-3-5/h1-3H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C4H5Cl/c1-3-4(2)5/h3H,1-2H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C6H10ClN3O3/c1-5(11)4-10(9-13)6(12)8-3-2-7/h2-4H2,1H3,(H,8,12):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C12H12N2O3/c1-2-12(8-6-4-3-5-7-8)9(15)13-11(17)14-10(12)16/h3-7H,2H2,1H3,(H2,13,14,15,16,17):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C6H12N2O4/c1-5(10)2-8(7-12)3-6(11)4-9/h6,9,11H,2-4H2,1H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C12H8Cl6O/c13-8-9(14)11(16)5-3-1-2(6-7(3)19-6)4(5)10(8,15)12(11,17)18/h2-7H,1H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C19H24N2O2/c22-18-13-20(19(23)15-7-2-1-3-8-15)12-17-16-9-5-4-6-14(16)10-11-21(17)18/h4-6,9,15,17H,1-3,7-8,10-13H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C6H10N2O2/c1-3-4-8(7-10)5-6(2)9/h3H,1,4-5H2,2H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C3H6ClNO/c1-5(2)3(4)6/h1-2H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C10H13NO2/c1-8(2)13-10(12)11-9-6-4-3-5-7-9/h3-8H,1-2H3,(H,11,12):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C2H4N4/c3-2-4-1-5-6-2/h1H,(H3,3,4,5,6):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C14H8Cl4/c15-11-5-1-9(2-6-11)13(14(17)18)10-3-7-12(16)8-4-10/h1-8H:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/H4N2/c1-2/h1-2H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C9H11N3O2/c13-10-12-6-2-4-9(12)8-3-1-5-11(14)7-8/h1,3,5,7,9H,2,4,6H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C5H13N3O/c1-7(2)4-5-8(3)6-9/h4-5H2,1-3H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/HNO2.Na/c2-1-3;/h(H,2,3);/q;+1/p-1:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C2HCl3/c3-1-2(4)5/h1H:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C5H12N2O4/c8-2-1-7(6-11)3-5(10)4-9/h5,8-10H,1-4H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C2H8N2/c1-4(2)3/h3H2,1-2H3:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C6H5NO2/c8-6(9)5-1-3-7-4-2-5/h1-4H,(H,8,9):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/CH2O/c1-2/h1H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C6H5N2.BF4/c7-8-6-4-2-1-3-5-6;2-1(3,4)5/h1-5H;/q+1;-1:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C27H30O16/c1-8-17(32)20(35)22(37)26(40-8)39-7-15-18(33)21(36)23(38)27(42-15)43-25-19(34)16-13(31)5-10(28)6-14(16)41-24(25)9-2-3-11(29)12(30)4-9/h2-6,8,15,17-18,20-23,26-33,35-38H,7H2,1H3/t8-,15+,17-,18+,20+,21-,22+,23+,26+,27?/m0/s1:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C6H10N2O/c1-3-5-8(7-9)6-4-2/h3-4H,1-2,5-6H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C2H3Cl/c1-2-3/h2H,1H2:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+ http://localhost/compound/InChI=1S/C14H14ClN3O2S/c1-8-4-3-5-10(9(8)2)16-12-6-11(15)17-14(18-12)21-7-13(19)20/h3-6H,7H2,1-2H3,(H,19,20)(H,16,17,18):
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false
+ http://localhost/compound/InChI=1S/C2H8N2.2ClH/c1-3-4-2;;/h3-4H,1-2H3;2*1H:
+ - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true
+features:
+- http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)
+title: Training dataset split of Hamster Carcinogenicity (DSSTOX/CPDB)
diff --git a/nightly/nightly.rb b/nightly/nightly.rb
index dc06ae3..594d4ee 100644
--- a/nightly/nightly.rb
+++ b/nightly/nightly.rb
@@ -1,4 +1,5 @@
+require 'test/test_examples.rb'
class Nightly
@@ -15,18 +16,15 @@ class Nightly
end
end
- def self.build_nightly(dry_run=false)
+ def self.build_nightly(select=nil, dry_run=false)
+
+ validationExamples = ValidationExamples.select(select)
+ return "please \"select\" validation examples:\n"+ValidationExamples.list if validationExamples.size==0
+
task_uri = OpenTox::Task.as_task() do
LOGGER.info("Building nightly report")
- benchmarks = [ HamsterTrainingTestBenchmark.new,
- #HamsterCrossvalidationBenchmark.new,
- #MiniRegressionBenchmark.new,
- #CacoModelsRegressionBenchmark.new,
- #CacoModelsRegressionBenchmark2.new,
- #CacoAlgsRegressionBenchmark.new,
- #FatheadRegressionBenchmark.new,
- ]
+ benchmarks = validationExamples.collect{ |e| ValidationBenchmark.new(e) }
running = []
report = Reports::XMLReport.new("Nightly Validation", Time.now.strftime("Created at %m.%d.%Y - %H:%M"))
@@ -60,15 +58,14 @@ class Nightly
section = report.add_section(report.get_root_element, b.title)
section_info = report.add_section(section, "Info")
- b.info.each{|i| report.add_paragraph(section_info,i)}
info_table = b.info_table
- report.add_table(section_info, b.info_table_title, info_table) if info_table
+ report.add_table(section_info, "Validation info", info_table) if info_table
section_results = report.add_section(section, "Results")
- report.add_table(section_results, b.result_table_title, b.result_table)
+ report.add_table(section_results, "Valdation results", b.result_table)
if (b.comparison_report)
- report.add_table(section_results, b.comparison_report_title, [[b.comparison_report]], false)
+ report.add_table(section_results, "Validation comparison report", [[b.comparison_report]], false)
end
section_errors = report.add_section(section, "Errors")
@@ -96,7 +93,7 @@ class Nightly
else
LOGGER.info("Nightly report completed - DRY RUN, no report creation")
end
- return "Nightly report completed"
+ benchmarks.collect{|b| b.uris}.join(",")
end
if defined?(halt)
halt 202,task_uri
@@ -105,120 +102,102 @@ class Nightly
end
end
- class AbstractBenchmark
-
- def info_table_title
- return title
+ class ValidationBenchmark
+
+ attr_accessor :errors, :comparison_report
+
+ def comparable
+ if @comp == nil
+ @comp = @validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri
+ end
+ @comp
end
- def info_table
- return nil
+ def uris
+ @validation_examples.collect{|v| v.validation_uri}.join(",")
+ end
+
+ def initialize(validationExamples)
+ @validation_examples = []
+ validationExamples.each do |v|
+ example = v.new
+ @validation_examples << example
+ end
end
- def result_table_title
- return "Validation results"
+ def title
+ @validation_examples.collect{|e| e.title}.join(" + ")
end
def result_table
- raise "no comparables" unless @comparables
- raise "no validations" unless @validations
- raise "no reports" unless @reports
t = []
- row = [comparable_nice_name, "validation", "report"]
+ row = [comparable.to_s, "validation", "report"]
t << row
- (0..@comparables.size-1).each do |i|
- row = [ @comparables[i], @validations[i], @reports[i] ]
+ @validation_examples.each do |e|
+ row = [ e.send(comparable),
+ (e.validation_error!=nil ? "error, see below" : e.validation_uri),
+ (e.report_error!=nil ? "error, see below" : e.report_uri) ]
t << row
end
t
end
-
- def comparison_report_title
- "algorithm comparison report"
- end
-
- def comparison_report
- @comparison_report
- end
-
- end
-
- class ValidationBenchmark < AbstractBenchmark
def info_table
- raise "no comparables" unless @comparables
t = []
t << ["param", "uri"]
- params.each do |k,v|
- t << [k.to_s, v.to_s]
- end
- count = 1
- @comparables.each do |alg|
- t << [comparable_nice_name+" ["+count.to_s+"]", alg]
- count += 1
+
+ (@validation_examples[0].params+@validation_examples[0].opt_params).each do |p|
+ map = {}
+ @validation_examples.each{|e| map[e.send(p).to_s]=nil }
+
+ if map.size==1 && map.keys[0].size==0
+ #omit
+ elsif map.size==1 #values equal
+ t << [p.to_s, map.keys[0]]
+ else
+ count = 1
+ @validation_examples.each do |e|
+ t << [p.to_s+" ["+count.to_s+"]", e.send(p)]
+ count += 1
+ end
+ end
end
t
end
- def errors
- @errors
- end
-
- def params
- raise "return uri-value hash"
- end
-
- def validate(index)
- raise "validate, return uri"
- end
-
- def build_report(index)
- raise "build report, return uri"
- end
-
- def build_compare_report(comparables)
- raise "build compare report, return uri"
- end
-
def build()
- raise "no comparables" unless @comparables
- @validations = Array.new(@comparables.size)
- @reports = Array.new(@comparables.size)
@errors = {}
- to_compare = []
-# LOGGER.info "train-data: "+@train_data.to_s
-# LOGGER.info "test-data: "+@test_data.to_s
-# LOGGER.info "test-class-data: "+@test_class_data.to_s
running = []
- (0..@comparables.size-1).each do |i|
-
+ count = 1
+
+ @validation_examples.each do |v|
+
+ id = v.title+count.to_s
+ count += 1
+ running << id
+ LOGGER.debug "Uploading datasets: "+v.title
+ v.upload_files
+ v.check_requirements
+
Thread.new do
- running << @comparables[i]+i.to_s
- begin
- LOGGER.debug "Validate: "+@comparables[i].to_s
- @validations[i] = validate(i)
- to_compare << @validations[i] if OpenTox::Utils.is_uri?(@validations[i])
-
- begin
- LOGGER.debug "Building validation-report for "+@validations[i].to_s+" ("+@comparables[i].to_s+")"
- @reports[i] = build_report(i)
- rescue => ex
- LOGGER.error "validation-report error: "+ex.message
- @reports[i] = "error"
+
+ LOGGER.debug "Validate: "+v.title
+ v.validate
+ if v.validation_error!=nil
+ @errors["Error validating "+v.title] = v.validation_error
+ else
+ LOGGER.debug "Building report: "+v.title
+ v.report
+ if v.report_error!=nil
+ @errors["Error building report for "+v.title] = v.report_error
end
-
- rescue => ex
- LOGGER.error "validation error: "+ex.message
- key = "Error validating "+@comparables[i].to_s
- @validations[i] = key+" (see below)"
- @errors[key] = ex.message
- ensure
- running.delete(@comparables[i]+i.to_s)
end
+ running.delete(id)
end
end
+
wait = 0
while running.size>0
LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+" waiting for "+running.inspect if wait%20==0
@@ -226,430 +205,9 @@ class Nightly
sleep 1
end
- if to_compare.size>1
- LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": build comparison report"
- @comparison_report = build_compare_report(to_compare)
- else
- LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": nothing to compare"
- end
- end
- end
-
- class AlgorithmValidationBenchmark < ValidationBenchmark
-
- def comparable_nice_name
- return "algorithm"
- end
-
- def build()
- raise "no algs" unless @algs
- @comparables = @algs
- super
- end
-
- def build_compare_report(comparables)
- Util.create_alg_comparison_report(comparables)
- end
- end
-
- class ModelValidationBenchmark < ValidationBenchmark
-
- def comparable_nice_name
- return "model"
- end
-
- def build()
- raise "no models" unless @models
- @comparables = @models
- super
- end
-
- def build_compare_report(comparables)
- "model comparsion report not available yet" #Util.create_model_comparison_report(comparables)
- end
- end
-
- class TestModelValidationBenchmark < ModelValidationBenchmark
-
- def info
- [ training_test_info ]
- end
-
- def training_test_info
- "This is a test set validation of existing models. "+
- "The model is used to predict the test dataset. Evaluation is done by comparing the model predictions "+
- "to the actual test values (in the test (target) dataset)."
- end
-
- def info_table_title
- "Validation params"
- end
-
- def params
- p = { "test_dataset_uri" => @test_data }
- p["test_target_dataset_uri"] = @test_class_data if @test_class_data
- return p
- end
-
- def validate(index)
- Util.validate_model(@test_data, @test_class_data, @models[index])
- end
-
- def build_report(index)
- Util.create_report(@validations[index])
- end
-
- def build()
- raise "no test data" unless @test_data
- super
- end
- end
-
- class TrainingTestValidationBenchmark < AlgorithmValidationBenchmark
-
- def info
- [ training_test_info ]
- end
-
- def training_test_info
- "This is a training test set validation. It builds a model with an algorithm and the training dataset. "+
- "The model is used to predict the test dataset. Evaluation is done by comparing the model predictions "+
- "to the actual test values (in the test target dataset)."
- end
-
- def info_table_title
- "Validation params"
- end
-
- def params
- p = { "training_dataset_uri" => @train_data, "test_dataset_uri" => @test_data,
- "prediction_feature" => @pred_feature }
- p["test_target_dataset_uri"] = @test_class_data if @test_class_data
- return p
- end
-
- def validate(index)
- Util.validate_alg(@train_data, @test_data, @test_class_data,
- @algs[index], @pred_feature, @alg_params[index])
- end
-
- def build_report(index)
- Util.create_report(@validations[index])
- end
-
- def build()
- raise "no train data" unless @train_data
- raise "no test data" unless @test_data
- raise "no pred feature" unless @pred_feature
- super
- end
- end
-
- class CrossValidationBenchmark < AlgorithmValidationBenchmark
-
- def info
- [ training_test_info ]
- end
-
- def training_test_info
- "This is a cross-validation."
- end
-
- def info_table_title
- "Cross-validation params"
+ LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": build comparison report"
+ @comparison_report = ValidationExamples::Util.build_compare_report(@validation_examples)
end
-
- def params
- p = { "dataset_uri" => @data, "prediction_feature" => @pred_feature,
- "num_folds" => @num_folds, "random_seed" => @random_seed, "stratified" => @stratified}
- return p
- end
-
- def validate(index)
- Util.cross_validate_alg(@data, @algs[index], @pred_feature,
- @num_folds, @random_seed, @stratified, @alg_params[index])
- end
-
- def build_report(index)
- Util.create_report(@validations[index], "crossvalidation")
- end
-
- def build()
- raise "no data" unless @data
- raise "no pred feature" unless @pred_feature
- @num_folds = 10 unless @num_folds
- @random_seed = 1 unless @random_seed
- @stratified = false unless @stratified
- super
- end
- end
-
- class HamsterCrossvalidationBenchmark < CrossValidationBenchmark
-
- @@dataset_service = @@config[:services]["opentox-dataset"]
- @@file=File.new("data/hamster_carcinogenicity.yaml","r")
- @@file_type="text/x-yaml"
- @@lazar_server = @@config[:services]["opentox-algorithm"]
-
- def title()
- "Classification, Crossvalidation, Hamster carcinogenicity"
- end
-
- def info
- res = [ "A crossvalidation using the hamster carcinogenicity dataset." ] + super
- return res
- end
-
- def build()
- @algs = [
- #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]),
- #File.join(@@lazar_server,"lazar"),
- "http://ot-test.in-silico.ch/algorithm/lazar",
- #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]),
- #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]),
- ]
- @alg_params = [
- #nil,
- #"feature_generation_uri="+File.join(@@lazar_server,"fminer"),
- "feature_generation_uri=http://ot-test.in-silico.ch/algorithm/fminer",
- nil,
- nil
- ]
- #@pred_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
- @pred_feature = "http://ot-test.in-silico.ch/toxcreate/feature#Hamster%20Carcinogenicity%20(CPDB/DSSTOX)"
- @data = "http://ot-test.in-silico.ch/dataset/1"
- #LOGGER.debug "upload hamster datasets"
- #@data = Util.upload_dataset(@@dataset_service, @@file, @@file_type).chomp("\n")
- super
- end
- end
-
- class HamsterTrainingTestBenchmark < TrainingTestValidationBenchmark
-
- @@dataset_service = @@config[:services]["opentox-dataset"]
- @@file=File.new("data/hamster_carcinogenicity.yaml","r")
- @@file_type="text/x-yaml"
- @@lazar_server = @@config[:services]["opentox-algorithm"]
-
- def title()
- "Classification, Algorithm training test validation, Hamster carcinogenicity"
- end
-
- def info
- res = [ "A simple binary classification task using the hamster carcinogenicity dataset." ] + super
- return res
- end
-
- def build()
- @algs = [
- #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]),
- File.join(@@lazar_server,"lazar"),
- #"http://188.40.32.88/algorithm/lazar",
- ]
- @alg_params = [
- #nil,
- "feature_generation_uri="+File.join(@@lazar_server,"fminer"),
- #"feature_generation_uri=http://188.40.32.88/algorithm/fminer",
- ]
-
- LOGGER.debug "prepare hamster datasets"
-
- @pred_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
- @test_class_data = Util.upload_dataset(@@dataset_service, @@file, @@file_type).chomp("\n")
- #@pred_feature = "http://188.40.32.88/toxcreate/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
- #@test_class_data = "http://188.40.32.88/dataset/57"
-
- split = Util.split_dataset(@test_class_data, @pred_feature, 0.9, 1)
- @train_data = split[0].to_s
- @test_data = split[1].to_s
- raise "could not split "+@train_data.to_s+" "+@test_data.to_s unless OpenTox::Utils.is_uri?(@train_data) and OpenTox::Utils.is_uri?(@test_data)
- super
- end
- end
-
-
- class MiniRegressionBenchmark < TrainingTestValidationBenchmark
-
- def title
- "Regression, Algorithm training test validation, tiny test dataset"
- end
-
- def info
- res = [ "A very small regression task, using the training dataset as test set." ] + super
- return res
- end
-
- def build()
- @algs = [
- "http://opentox.ntua.gr:3003/algorithm/mlr",
- "http://opentox.ntua.gr:3004/algorithm/mlr",
- "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNregression",
- File.join(@@config[:services]["opentox-majority"],["/regr/algorithm"])
- ]
- @alg_params = [ nil, "dataset_service=http://ambit.uni-plovdiv.bg:8080/ambit2/dataset", nil]
- @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
- super
- end
- end
-
- class CacoAlgsRegressionBenchmark < TrainingTestValidationBenchmark
-
- def title
- "Regression, Algorithm training test validation, Caco-2 dataset"
- end
-
- def info
- res = [ "Training test set validation on caco2 dataset." ] + super
- return res
- end
-
- def build()
- @algs = [
- "http://opentox.ntua.gr:3003/algorithm/mlr",
- "http://opentox.ntua.gr:3004/algorithm/mlr",
- "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/LR",
- ]
- @alg_params = [ nil, nil]
- @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798"
- @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353"
- @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510"
- super
- end
- end
-
-
- class CacoModelsRegressionBenchmark < TestModelValidationBenchmark
-
- def title
- "Regression, Model test set validation, Caco-2 dataset (ambit.uni-plovdiv)"
- end
-
- def info
- res = [ "Valdation of two identical(?) mlr models on caco-2 dataset." ] + super
- return res
- end
-
- def build()
- @models = [
- "http://ambit.uni-plovdiv.bg:8080/ambit2/model/259260",
- "http://opentox.ntua.gr:3003/model/195",
- ]
- #@test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798"
- @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353"
- super
- end
- end
-
-
- class CacoModelsRegressionBenchmark2 < TestModelValidationBenchmark
-
- def title
- "Regression, Model test set validation, Caco-2 dataset (apps.ideaconsult)"
- end
-
- def info
- res = [ "One more caco2 dataset at another location" ] + super
- return res
- end
-
- def build()
- @models = [
- "http://apps.ideaconsult.net:8080/ambit2/model/33",
- ]
- @test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
- #prediction_feature=http://apps.ideaconsult.net:8080/ambit2/feature/22200
- super
- end
- end
-
-
-
- class FatheadRegressionBenchmark < TrainingTestValidationBenchmark
-
- def title
- "Training test set validation, regression with fathead minnow dataset"
- end
-
- def info
- res = [ "This is the regression use case used in D2.2. "+
- "The task is to predict LC50 values of the well known Fathead Minnow Acute Toxicity dataset. "+
- "JOELIB was used to compute numerical descriptors as features." ] + super
- return res
- end
-
- def build()
- @algs = [
- "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNregression",
- "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/M5P",
- "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/GaussP"
- ]
- @alg_params = [nil, nil, nil]
- @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/639"
- @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/640"
- @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/264185"
- super
- end
- end
-
-
-
-
- class Util
- @@validation_service = @@config[:services]["opentox-validation"]
-
- def self.upload_dataset(dataset_service, file, file_type)
- raise "File not found: "+file.path.to_s unless File.exist?(file.path)
- data = File.read(file.path)
- data_uri = OpenTox::RestClientWrapper.post dataset_service, {:content_type => file_type}, data
- #data_uri = OpenTox::Task.find(data_uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(data_uri)
- return data_uri.to_s
- end
-
- def self.split_dataset(data_uri, feature, split_ratio, random_seed)
- res = OpenTox::RestClientWrapper.post File.join(@@validation_service,'plain_training_test_split'), { :dataset_uri => data_uri, :prediction_feature=>feature, :split_ratio=>split_ratio, :random_seed=>random_seed}
- return res.split("\n")
- end
-
- def self.validate_alg(train_data, test_data, test_class_data, alg, feature, alg_params)
- uri = OpenTox::RestClientWrapper.post @@validation_service, { :training_dataset_uri => train_data, :test_dataset_uri => test_data,
- :test_target_dataset_uri => test_class_data,
- :algorithm_uri => alg, :prediction_feature => feature, :algorithm_params => alg_params }
- #LOGGER.info "waiting for validation "+uri.to_s
- #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
- #LOGGER.info "validaiton done "+uri.to_s
- return uri.to_s
- end
-
- def self.validate_model(test_data, test_class_data, model)
- uri = OpenTox::RestClientWrapper.post @@validation_service, { :test_dataset_uri => test_data,
- :test_target_dataset_uri => test_class_data, :model_uri => model }
- return uri.to_s
- end
-
- def self.cross_validate_alg(data, alg, feature, folds, seed, stratified, alg_params)
- uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"crossvalidation"), { :dataset_uri => data,
- :algorithm_uri => alg, :prediction_feature => feature, :algorithm_params => alg_params, :num_folds => folds,
- :random_seed => seed, :stratified => stratified }
- #LOGGER.info "waiting for validation "+uri.to_s
- #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
- #LOGGER.info "validaiton done "+uri.to_s
- return uri.to_s
- end
-
- def self.create_report(validation, type="validation")
- uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"report/"+type.to_s), { :validation_uris => validation }
- #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
- return uri.to_s
- end
-
- def self.create_alg_comparison_report(validations)
- uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"report/algorithm_comparison"), { :validation_uris => validations.join("\n") }
- #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri)
- return uri.to_s
- end
-
end
end \ No newline at end of file
diff --git a/nightly/nightly_application.rb b/nightly/nightly_application.rb
index c366024..69b4614 100644
--- a/nightly/nightly_application.rb
+++ b/nightly/nightly_application.rb
@@ -1,8 +1,8 @@
require "nightly/nightly.rb"
-get '/build_nightly/?' do
+post '/build_nightly/?' do
dry_run = params[:dry_run]!=nil and params[:dry_run]
- Nightly.build_nightly(dry_run)
+ Nightly.build_nightly(params["select"],dry_run)
end
get '/css_style_sheet/?' do
diff --git a/report/report_test.rb b/report/report_test.rb
index 7a884b1..d580045 100644
--- a/report/report_test.rb
+++ b/report/report_test.rb
@@ -9,6 +9,7 @@ LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S "
class Reports::ApplicationTest < Test::Unit::TestCase
include Rack::Test::Methods
+ include Lib::TestUtil
def app
Sinatra::Application
@@ -37,10 +38,13 @@ class Reports::ApplicationTest < Test::Unit::TestCase
#post 'http://ot.validation.de/report/crossvalidation',:validation_uris=>"http://ot.validation.de/crossvalidation/1"
#uri = last_response.body.to_s
- val_uris = ["http://ot.validation.de/crossvalidation/7","http://ot.validation.de/crossvalidation/8" ]
+ val_uris = ["http://ot.validation.de/crossvalidation/32"] #,"http://ot.validation.de/crossvalidation/8" ]
- post 'http://ot.validation.de/report/algorithm_comparison',:validation_uris=>val_uris.join("\n")
- uri = last_response.body.to_s
+ post 'http://ot.validation.de/report/crossvalidation',:validation_uris=>val_uris.join("\n")
+ uri = wait_for_task(last_response.body.to_s)
+ puts uri
+ id = uri.squeeze("/").split("/")[-1]
+ get '/report/crossvalidation/'+id,nil,'HTTP_ACCEPT' => "text/html"
puts uri
#rep = Reports::ReportService.new("http://some.location")
diff --git a/test/test_application.rb b/test/test_application.rb
new file mode 100644
index 0000000..cbc1057
--- /dev/null
+++ b/test/test_application.rb
@@ -0,0 +1,17 @@
+
+post '/test_validation/?' do
+ validationExamples = ValidationExamples.select(params[:select])
+ return "please \"select\" a single validation example:\n"+ValidationExamples.list if validationExamples.size!=1 or validationExamples[0].size!=1
+ OpenTox::Task.as_task do
+ v = validationExamples[0][0]
+ ex = v.new
+ ex.upload_files
+ ex.check_requirements
+ ex.validate
+ raise ex.validation_error if ex.validation_error
+ ex.report if params[:report]
+ raise ex.report_error if ex.report_error
+ ex.validation_uri + (params[:report] ? ","+ex.report_uri : "")
+ end
+end
+
diff --git a/test/test_examples.rb b/test/test_examples.rb
new file mode 100644
index 0000000..d08e358
--- /dev/null
+++ b/test/test_examples.rb
@@ -0,0 +1,214 @@
+
+require 'test/test_examples_util.rb'
+
+class Class
+ def humanize
+ self.to_s.gsub(/.*::/, "").gsub(/([^^A-Z_])([A-Z])/, '\1-\2').gsub(/_/,"-")
+ end
+end
+
+module ValidationExamples
+
+ ########################################################################################################
+
+ class ABSTRACTHamsterSplit < SplitTestValidation
+ def initialize
+ @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
+ @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+ end
+ end
+
+ class LazarHamsterSplit < ABSTRACTHamsterSplit
+ def initialize
+ @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
+ @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+ super
+ end
+ end
+
+ class MajorityHamsterSplit < ABSTRACTHamsterSplit
+ def initialize
+ @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+ super
+ end
+ end
+
+ ########################################################################################################
+
+ class ABSTRACTHamsterTrainingTest < TrainingTestValidation
+ def initialize
+ @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
+ @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r")
+ @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r")
+ @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+ end
+ end
+
+ class MajorityHamsterTrainingTest < ABSTRACTHamsterTrainingTest
+ def initialize
+ @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+ super
+ end
+ end
+
+ class LazarHamsterTrainingTest < ABSTRACTHamsterTrainingTest
+ def initialize
+ @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
+ @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+ super
+ end
+ end
+
+ ########################################################################################################
+
+ class ABSTRACTHamsterCrossvalidation < CrossValidation
+ def initialize
+ @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r")
+ @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+ end
+ end
+
+ class MajorityHamsterCrossvalidation < ABSTRACTHamsterCrossvalidation
+ def initialize
+ @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+ super
+ end
+ end
+
+ class LazarHamsterCrossvalidation < ABSTRACTHamsterCrossvalidation
+ def initialize
+ @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
+ @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+ super
+ end
+ end
+
+ ########################################################################################################
+
+ class ABSTRACT_ISTHamsterCrossvalidationInsilico < CrossValidation
+ def initialize
+ @dataset_uri = "http://webservices.in-silico.ch/dataset/108"
+ @prediction_feature = "http://toxcreate.org/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
+ end
+ end
+
+ class MajorityISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico
+ def initialize
+ @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm")
+ super
+ end
+ end
+
+ class LazarISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico
+ def initialize
+ @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar")
+ @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
+ super
+ end
+ end
+
+ class ISTLazarISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico
+ def initialize
+ @algorithm_uri = "http://webservices.in-silico.ch/algorithm/lazar"
+ @algorithm_params = "feature_generation_uri=http://webservices.in-silico.ch/algorithm/fminer"
+ super
+ end
+ end
+
+
+ ########################################################################################################
+
+ class LR_AmbitCacoModel < ModelValidation
+ def initialize
+ @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/33"
+ @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545"
+ #@prediction_feature=http://apps.ideaconsult.net:8080/ambit2/feature/22200
+ end
+ end
+
+ ########################################################################################################
+
+ class ABSTRACTCacoTrainingTest < TrainingTestValidation
+ def initialize
+ @training_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798"
+ @test_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353"
+ @prediction_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510"
+ end
+ end
+
+ class LR_AmbitCacoTrainingTest < ABSTRACTCacoTrainingTest
+ def initialize
+ @algorithm_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/LR"
+ super
+ end
+ end
+
+ class MLR_NTUA_CacoTrainingTest < ABSTRACTCacoTrainingTest
+ def initialize
+ @algorithm_uri = "http://opentox.ntua.gr:3003/algorithm/mlr"
+ super
+ end
+ end
+
+ class MLR_NTUA2_CacoTrainingTest < ABSTRACTCacoTrainingTest
+ def initialize
+ @algorithm_uri = "http://opentox.ntua.gr:3004/algorithm/mlr"
+ super
+ end
+ end
+
+ class MajorityCacoTrainingTest < ABSTRACTCacoTrainingTest
+ def initialize
+ @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm")
+ super
+ end
+ end
+
+ ########################################################################################################
+
+ @@list = {
+ "1" => [ LazarHamsterSplit, MajorityHamsterSplit ],
+ "1a" => [ LazarHamsterSplit ],
+ "1b" => [ MajorityHamsterSplit ],
+
+ "2" => [ LazarHamsterTrainingTest, MajorityHamsterTrainingTest ],
+ "2a" => [ LazarHamsterTrainingTest ],
+ "2b" => [ MajorityHamsterTrainingTest ],
+
+ "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation ],
+ "3a" => [ LazarHamsterCrossvalidation ],
+ "3b" => [ MajorityHamsterCrossvalidation ],
+
+ "4" => [ MajorityISTHamsterCrossvalidation, LazarISTHamsterCrossvalidation, ISTLazarISTHamsterCrossvalidation ],
+ "4a" => [ MajorityISTHamsterCrossvalidation ],
+ "4b" => [ LazarISTHamsterCrossvalidation ],
+ "4c" => [ ISTLazarISTHamsterCrossvalidation ],
+
+ "5a" => [ LR_AmbitCacoModel ],
+
+ "6" => [ LR_AmbitCacoTrainingTest, MLR_NTUA_CacoTrainingTest, MLR_NTUA2_CacoTrainingTest, MajorityCacoTrainingTest ],
+ "6a" => [ LR_AmbitCacoTrainingTest ],
+ "6b" => [ MLR_NTUA_CacoTrainingTest ],
+ "6c" => [ MLR_NTUA2_CacoTrainingTest ],
+ "6d" => [ MajorityCacoTrainingTest ],
+ }
+
+ def self.list
+ @@list.sort.collect{|k,v| k+":\t"+v.collect{|vv| vv.humanize}.join("\n\t")+"\n"} #.join("\n")
+ end
+
+ def self.select(csv_keys)
+ res = []
+ if csv_keys!=nil and csv_keys.size>0
+ csv_keys.split(",").each do |k|
+ raise "no key "+k.to_s unless @@list.has_key?(k)
+ res << @@list[k]
+ end
+ end
+ return res
+ end
+
+end
+
+#puts ValidationExamples.list
+#puts ValidationExamples.select("1,2a").inspect \ No newline at end of file
diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb
new file mode 100644
index 0000000..49e0336
--- /dev/null
+++ b/test/test_examples_util.rb
@@ -0,0 +1,194 @@
+
+module ValidationExamples
+
+ class Util
+
+ @@dataset_uris = {}
+
+ def self.upload_dataset(file, dataset_service=@@config[:services]["opentox-dataset"], file_type="text/x-yaml")
+ raise "File not found: "+file.path.to_s unless File.exist?(file.path)
+ if @@dataset_uris[file.path.to_s]==nil
+ data = File.read(file.path)
+ data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp
+ @@dataset_uris[file.path.to_s] = data_uri
+ LOGGER.debug "uploaded dataset: "+data_uri
+ else
+ LOGGER.debug "file already uploaded: "+@@dataset_uris[file.path.to_s]
+ end
+ return @@dataset_uris[file.path.to_s]
+ end
+
+ def self.build_compare_report(validation_examples)
+
+ @comp = validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri
+ return nil if @comp == :model_uri
+ to_compare = []
+ validation_examples.each do |v|
+ to_compare << v.validation_uri if v.validation_uri and v.validation_error==nil
+ end
+ return nil if to_compare.size < 2
+ begin
+ return validation_post "report/algorithm_comparison",{ :validation_uris => to_compare.join("\n") }
+ rescue => ex
+ return "error creating comparison report "+ex.message
+ end
+ end
+
+ def self.validation_post(uri, params)
+ if $test_case
+ #puts "posting: "+uri+","+params.inspect
+ $test_case.post uri,params
+ return wait($test_case.last_response.body)
+ else
+ return OpenTox::RestClientWrapper.post(File.join(@@config[:services]["opentox-validation"],uri),params)
+ end
+ end
+
+ def self.wait(uri)
+ if OpenTox::Utils.task_uri?(uri)
+ task = OpenTox::Task.find(uri)
+ task.wait_for_completion
+ raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error?
+ uri = task.resultURI
+ end
+ uri
+ end
+
+ end
+
+ class ValidationExample
+
+ #params
+ attr_accessor :name,
+ :prediction_feature,
+ :algorithm_uri,
+ :model_uri,
+ :test_dataset_uri,
+ :test_dataset_file,
+ :test_target_dataset_uri,
+ :test_target_dataset_file,
+ :training_dataset_uri,
+ :training_dataset_file,
+ :dataset_uri,
+ :dataset_file,
+ :algorithm_params,
+ :split_ratio,
+ :random_seed,
+ :num_folds,
+ :stratified
+ #results
+ attr_accessor :validation_uri,
+ :report_uri,
+ :validation_error,
+ :report_error
+
+ def upload_files
+ [[:test_dataset_uri, :test_dataset_file],
+ [:test_target_dataset_uri, :test_target_dataset_file],
+ [:training_dataset_uri, :training_dataset_file],
+ [:dataset_uri, :dataset_file]].each do |a|
+ uri = a[0]
+ file = a[1]
+ if send(uri)==nil and send(file)!=nil
+ send("#{uri.to_s}=".to_sym, Util.upload_dataset(send(file)))
+ end
+ end
+ end
+
+ def check_requirements
+ params.each do |r|
+ raise "values not set: "+r.to_s if send(r)==nil
+ end
+ end
+
+ def report
+ begin
+ @report_uri = Util.validation_post '/report/'+report_type,{:validation_uris => @validation_uri}
+ rescue => ex
+ @report_error = ex.message
+ end
+ end
+
+ def validate
+ begin
+ @validation_uri = Util.validation_post '/'+validation_type, get_params
+ rescue => ex
+ @validation_error = ex.message
+ end
+ end
+
+ def title
+ self.class.humanize
+ end
+
+ protected
+ def report_type
+ "validation"
+ end
+
+ def validation_type
+ ""
+ end
+
+ def get_params
+ p = {}
+ ( params + opt_params ).each do |pp|
+ p[pp] = send(pp) if send(pp)!=nil
+ end
+ return p
+ end
+ end
+
+ class ModelValidation < ValidationExample
+
+ def params
+ [:model_uri, :test_dataset_uri]
+ end
+
+ def opt_params
+ [ :prediction_feature, :test_target_dataset_uri ]
+ end
+ end
+
+ class TrainingTestValidation < ValidationExample
+ def params
+ [:algorithm_uri, :training_dataset_uri, :test_dataset_uri, :prediction_feature]
+ end
+
+ def opt_params
+ [ :algorithm_params, :test_target_dataset_uri ]
+ end
+ end
+
+ class SplitTestValidation < ValidationExample
+ def params
+ [:algorithm_uri, :dataset_uri, :prediction_feature]
+ end
+
+ def opt_params
+ [ :algorithm_params, :split_ratio, :random_seed ]
+ end
+
+ def validation_type
+ "training_test_split"
+ end
+ end
+
+ class CrossValidation < ValidationExample
+ def params
+ [:algorithm_uri, :dataset_uri, :prediction_feature]
+ end
+
+ def opt_params
+ [ :algorithm_params, :num_folds, :stratified, :random_seed ]
+ end
+
+ def report_type
+ "crossvalidation"
+ end
+
+ def validation_type
+ "crossvalidation"
+ end
+ end
+end \ No newline at end of file
diff --git a/validation/validation_test.rb b/validation/validation_test.rb
index fbf8d78..0d59c42 100644
--- a/validation/validation_test.rb
+++ b/validation/validation_test.rb
@@ -6,635 +6,45 @@ require 'application.rb'
require 'test/unit'
require 'rack/test'
require 'lib/test_util.rb'
+require 'test/test_examples.rb'
LOGGER = Logger.new(STDOUT)
LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S "
-class Example
- attr_accessor :alg, :train_data, :test_data, :model, :pred_data, :act_feat, :pred_feat, :classification, :alg_params, :val, :orig_data, :num_folds, :random_seed
-end
-
class ValidationTest < Test::Unit::TestCase
include Rack::Test::Methods
include Lib::TestUtil
def test_it
- #Nightly.build_nightly
- #get "/build_nightly"
- #get "/nightly"
- #get '20',nil,'HTTP_ACCEPT' => "application/rdf+xml"
- #puts last_response.body
-
- #prepare_examples
- #do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE
-
- #ex = ex_maj_class
- #ex = ex_maj_regr
- #ex = ex_ntua
- #ex = ex_ntua2
- #ex = ex_tum
- #ex = ex_tum2
- #ex = ex_local
- ex = ex_lazar_extern
- #ex = ex_ambit
-
- #create_validation(ex)
- #validate_model(ex)
- validate_algorithm(ex)
- #validate_split(ex)
- #xval(ex)
-
- #test_dataset = OpenTox::Dataset.find ex_ntua2.pred_data
- #puts ex_ntua2.pred_data.to_s+", "+test_dataset.compounds.size.to_s+" compounds"
- #puts test_dataset.compounds.collect{|c| c.to_s}.join("\n")
- end
-
- def app
- Sinatra::Application
- end
-
- def ex_maj_regr
- ex = Example.new
- ex.classification = true
-
- ex.alg = File.join(@@config[:services]["opentox-majority"],"regr/algorithm")
- #ex.alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
-
- ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
-
- return ex
- end
-
- def ex_maj_class
- ex = Example.new
- ex.classification = true
-
- ex.alg = File.join(@@config[:services]["opentox-majority"],"class/algorithm")
- ex.alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
-
- get_hamster(ex)
- return ex
- end
-
- def get_hamster(ex)
- dataset = @@config[:services]["opentox-dataset"]
- ex.orig_data = File.join(dataset,"1")
- begin
- orig = OpenTox::Dataset.find(ex.orig_data)
- raise "not correct, upload" if (orig == nil || orig.compounds.size!=85)
- rescue => e
- puts e.message
- upload_uri = upload_data(dataset, File.new("data/hamster_carcinogenicity.yaml","r"))
- ex.orig_data = upload_uri
- end
- #ex.act_feat = "http://localhost/toxmodel/feature#Hamster Carcinogenicity (DSSTOX/CPDB)"
- ex.act_feat = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)"
-
- ex.train_data = File.join(dataset,"2")
- ex.test_data = File.join(dataset,"3")
- begin
- train = OpenTox::Dataset.find(ex.train_data)
- test = OpenTox::Dataset.find(ex.test_data)
- raise "not correct, split "+train.to_s+" "+test.to_s+
- " "+train.compounds.size.to_s+" "+test.compounds.size.to_s if (train == nil || test == nil || train.compounds.size>=85 || test.compounds.size>=train.compounds.size)
- rescue => e
- puts e.message
- post '/plain_training_test_split', { :dataset_uri => ex.orig_data, :prediction_feature=>ex.act_feat, :split_ratio=>0.75, :random_seed=>6}
- split = last_response.body.split("\n")
- ex.train_data = split[0]
- ex.test_data = split[1]
- end
- end
-
- def ex_local
- ex = Example.new
- ex.classification = true
-
- ex.alg = File.join(@@config[:services]["opentox-algorithm"],"lazar")
- ex.alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer")
-
- get_hamster(ex)
-
- #{:orig => ex.orig_data ,:train => ex.train_data, :test=> ex.test_data}.each do |k,v|
- # puts k.to_s+": "+v
- # OpenTox::Dataset.find(v).compounds.each do |c|
- # puts "XX "+c.to_s if c.to_s =~ /C6H12/
- # end
- #end
- ex.model = "http://ot.model.de/9"
- # example model
- #ex.model = "http://opentox.ntua.gr:3000/model/29"
- #ex.pred_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/261687"
- # example prediction data
- #ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/625"
- return ex
- end
-
- def ex_lazar_extern
- #server = "http://188.40.32.88/"
- server = "http://ot-test.in-silico.ch/"
-
- ex = Example.new
- ex.classification = true
- ex.alg = server+"algorithm/lazar"
- ex.train_data = server+"dataset/1" #51
- ex.test_data = ex.train_data
- ex.orig_data = ex.train_data
- #ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53
-
- ex.alg_params = "feature_generation_uri="+server+"algorithm/fminer"
-
- ex.act_feat = server+"toxcreate/feature#Hamster%20Carcinogenicity%20(CPDB/DSSTOX)"
-
-
- #ex.act_feat = server+"toxcreate/feature#feature#Hamster Carcinogenicity (DSSTOX/CPDB)"
-
- #ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53
- #ex.model = "http://apps.ideaconsult.net:8080/ambit2/model/20"
- return ex
- end
-
- def ex_ambit
- ex = Example.new
- ex.classification = false
-
- #ex.alg = "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/pka"
- #ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- #ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
-
-
- ex.model = "http://ambit.uni-plovdiv.bg:8080/ambit2/model/259260"
- ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798"
- #ex. http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510
-
- #ex.alg = "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/pka"
- #ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- #ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
-
- #ex.alg = "http://apps.ideaconsult.net:8080/ambit2/algorithm/pka"
- #ex.train_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/54" #53
- #ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53
- #ex.act_feat = "http://apps.ideaconsult.net:8080/ambit2/feature/22200" #22190"
- #ex.model = "http://apps.ideaconsult.net:8080/ambit2/model/20"
-
- return ex
- end
-
- def ex_ntua2
- ex = Example.new
- ex.classification = false
- ex.alg = "http://opentox.ntua.gr:3000/algorithm/mlr"
- ex.train_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/54" #53
- ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53
- ex.act_feat = "http://apps.ideaconsult.net:8080/ambit2/feature/22200" #22190"
-
-
- # example model
- #ex.model = "http://opentox.ntua.gr:3000/model/29"
- #ex.pred_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/261687"
- # example prediction data
- #ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/625"
- return ex
- end
-
- def ex_ntua
- ex = Example.new
- ex.classification = false
- ex.alg = "http://opentox.ntua.gr:3003/algorithm/mlr"
-
- #ex.orig_data = "http://apps.ideaconsult.net:8180/ambit2/dataset/52"
-
- ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
-
- ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
-
- # example model
- ex.model = "http://opentox.ntua.gr:3003/model/201"
- #ex.pred_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/264783"
- # example prediction data
- #ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/656"
- # validation
- #ex.val = "http://ot.validation.de/21"
- return ex
- end
-
- def ex_tum
- ex = Example.new
- ex.classification = false
- ex.alg = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNregression"
- #ex.alg = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/M5P"
- #ex.alg = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/GaussP"
-
- #mini
- ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
-
- #big
- #ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/639"
- #ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/640"
- #ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/264185"
- #ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/264187" #test
-
- # example model
- ex.model = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/model/TUMOpenToxModel_kNN_23"
- #ex.model = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/model/TUMOpenToxModel_M5P_13"
- #ex.model = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/model/TUMOpenToxModel_GaussP_4"
-
- #ex.pred_feat =
-
- # example prediction data
- ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/659" #knn
- #ex.pred_data = http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/658" #m5p
- #ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/653" #gaus
-
- # validataion
- # ex.val = "http://ot.validation.de/34"
- return ex
- end
-
-def ex_tum2
- ex = Example.new
- ex.classification = false
- ex.alg = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/J48"
- ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/1027"
- ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/1027"
- ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/357709"
- return ex
- end
-
-# def test_all_validations
-# get '/'
-# assert last_response.ok?
-# puts "all validations:\n"+last_response.body
-# validations = last_response.body.split("\n")
-# puts validations.inspect
-# if validations && validations.size>0
-# validations.each do |v|
-# get v
-# puts "get "+v+":\n"+last_response.body
-# assert last_response.ok? || last_response.status==202
-# end
-# end
-# end
-#
-# def test_all_cvs
-# get '/crossvalidations'
-# assert last_response.ok?
-# puts "all crossvalidations:\n"+last_response.body+"\n"
-# crossvalidations = last_response.body.split("\n")
-# #puts crossvalidations.inspect
-# if crossvalidations && crossvalidations.size>0
-# crossvalidations.each do |v|
-# get v
-# puts "get "+v+":\n"+last_response.body
-# assert last_response.ok? || last_response.status==202
-# end
-# end
-# end
-#
- def xval(ex)
- begin
- #data_uri = upload_data(WS_DATA, FILE)
-
-# first_validation=nil
-# 2.times do
-
- raise "no orig data" unless ex.orig_data
- num_folds = ex.num_folds ? ex.num_folds : 3
- random_seed = ex.random_seed ? ex.random_seed : 1
-
- post '/crossvalidation', { :dataset_uri => ex.orig_data, :algorithm_uri => ex.alg, :prediction_feature => ex.act_feat,
- :algorithm_params => ex.alg_params, :num_folds => num_folds, :random_seed => random_seed }
-
- uri = last_response.body
- uri = wait_for_task(uri)
- puts "crossvalidation: "+uri
-
- assert last_response.ok?
- crossvalidation_id = uri.split("/")[-1]
- add_resource("/crossvalidation/"+crossvalidation_id)
- puts "id:"+crossvalidation_id
-
- get '/crossvalidation/'+crossvalidation_id
- puts last_response.body
- #assert last_response.ok? || last_response.status==202
-
- #get '/crossvalidation/'+crossvalidation_id+'/validations'
- #puts "validations:\n"+last_response.body
- #assert last_response.ok?
- # assert last_response.body.split("\n").size == num_folds, "num-folds:"+num_folds.to_s+" but num lines is "+last_response.body.split("\n").size.to_s
-
-# if first_validation
-# # assert that both cross validaitons use the same datasets
-# first_validation2 = last_response.body.split("\n")[0].split("/")[-1]
-#
-# get '/'+first_validation+'/test_dataset_uri'
-# assert last_response.ok?
-# first_val_test_data = last_response.body
-#
-# get '/'+first_validation2+'/test_dataset_uri'
-# assert last_response.ok?
-# first_val2_test_data = last_response.body
-# assert first_val_test_data==first_val2_test_data
-# end
-# first_validation = last_response.body.split("\n")[0].split("/")[-1]
-# end
- ensure
- #delete_resources
- end
- end
-#
- def validate_model(ex)
- begin
-# data_uri_train = upload_data(WS_DATA, DATA_TRAIN, FILE_TRAIN)
-# data_uri_test = upload_data(WS_DATA, DATA_TEST, FILE_TEST)
-# #data_uri_train = WS_DATA+"/"+DATA_TRAIN
-# #data_uri_test = WS_DATA+"/"+DATA_TEST
-#
-# if WS_FEATURE_ALG
-# feature_uri = RestClient.post WS_FEATURE_ALG, :dataset_uri => data_uri_train
-# model_uri = RestClient.post(WS_CLASS_ALG,{ :activity_dataset_uri => data_uri_train, :feature_dataset_uri => feature_uri })
-# else
-# model_uri = RestClient.post(WS_CLASS_ALG,{ :dataset_uri => data_uri_train })
-# end
-
-# model_uri = "http://ot.model.de/1"
-# data_uri_test = "http://ot.dataset.de/3"
-
- #model_uri = "http://ot.model.de/7"
- #data_uri_test = "http://ot.dataset.de/41"
-
- # model_uri = "http://opentox.ntua.gr:3000/model/9"
- # data_uri_test = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
-
- raise "model not defined" unless ex.model
-
- post '', {:test_dataset_uri => ex.test_data,
- :test_target_dataset_uri => ex.orig_data,
- :model_uri => ex.model} #, :prediction_feature => FEATURE_URI}
-
- puts last_response.body
- #verify_validation
-
- val_uri = wait_for_task(last_response.body)
- puts val_uri
-
- #get val_uri
- #verify_validation(last_response.body)
-
- ensure
- #delete_resources
- end
- end
-
- def validate_algorithm(ex)
- begin
-
- #get '/41',nil,'HTTP_ACCEPT' => "application/rdf+xml" #"text/x-yaml"
- #puts last_response.body
-
- #data_uri_train = upload_data(WS_DATA, FILE_TRAIN)
- #data_uri_test = upload_data(WS_DATA, FILE_TEST)
-
- #data_uri_train = WS_DATA+"/"+DATA_TRAIN
- #data_uri_test = WS_DATA+"/"+DATA_TEST
-
-
-# data_uri_train="http://ot.dataset.de/57"
-# data_uri_test="http://ot.dataset.de/56"
-# feature_uri = FEATURE_URI
-# algorithm_uri = WS_CLASS_ALG
-# algorithm_params="feature_generation_uri="+WS_FEATURE_ALG
-
- #data_uri_train="http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
- #data_uri_test=data_uri_train
- #feature_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
- #algorithm_uri = "http://opentox.ntua.gr:3000/algorithm/mlr"
- #algorithm_params=nil
-
- #post '', { :training_dataset_uri => data_uri_train, :test_dataset_uri => data_uri_test,
- #:algorithm_uri => algorithm_uri, :prediction_feature => feature_uri, :algorithm_params => algorithm_params }
-
-# uri = OpenTox::RestClientWrapper.post(@@config[:services]["opentox-validation"],{ :training_dataset_uri => ex.train_data,
-# :test_dataset_uri => ex.test_data,
-# :test_target_dataset_uri => ex.orig_data,
-# :algorithm_uri => ex.alg,
-# :prediction_feature => ex.act_feat,
-# :algorithm_params => ex.alg_params
-# },nil,true)
-
- post '', { :training_dataset_uri => ex.train_data, :test_dataset_uri => ex.test_data, :test_target_dataset_uri => ex.orig_data,
- :algorithm_uri => ex.alg, :prediction_feature => ex.act_feat, :algorithm_params => ex.alg_params }
- uri = last_response.body.to_s.chomp("\n")
- puts uri
- uri = wait_for_task(uri)
-
- puts uri
- #get uri
- #verify_validation(last_response.body)
- #verify_validation
- ensure
- #delete_resources
- end
- end
-
- def create_validation(ex)
-
-# classification = false
-# test_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342"
-# prediction_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/407"
-# actual_feature="http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141"
-# predicted_feature = OpenTox::Model::PredictionModel.find("http://opentox.ntua.gr:3000/model/9").predicted_variables
-# assert predicted_feature=="http://ambit.uni-plovdiv.bg:8080/ambit2/feature/227289","nope: "+predicted_feature.to_s
-# #predicted_feature="http://ambit.uni-plovdiv.bg:8080/ambit2/feature/227289"
-
-# classification = false
-# test_dataset_uri = EX2_TEST
-# prediction_dataset_uri = EX2_PRED
-# actual_feature= EX2_ACT_FEAT
-# predicted_feature = OpenTox::Model::PredictionModel.find(EX2_MODEL).predicted_variables
-# assert predicted_feature==EX2_PRED_FEAT,"nope: "+predicted_feature.to_s
-
-# classification = true
-# test_dataset_uri = "http://ot.dataset.de/1"
-# prediction_dataset_uri = "http://ot.dataset.de/5"
-# actual_feature=URI.encode(FEATURE_URI)
-# predicted_feature = OpenTox::Model::PredictionModel.find("http://ot.model.de/1").predicted_variables#_lazar_classification
-# assert predicted_feature==URI.encode(FEATURE_URI+"_lazar_classification"), predicted_feature
-# #predicted_feature="http://www.epa.gov/NCCT/dsstox/CentralFieldDef.html#ActivityOutcome_CPDBAS_Hamster_lazar_prediction"
-
- #puts Lib::OTPredictions.new( classification, test_dataset_uri, actual_feature, prediction_dataset_uri, predicted_feature ).compute_stats.each{|key,value| puts key.to_s+" => "+value.to_s }
-
-
-
- #predicted_feature = OpenTox::Model::PredictionModel.find(ex.model).predicted_variables
- #puts "predicted feature "+predicted_feature.to_s
-
- assert(predicted_feature==ex.pred_feat,"nope: "+predicted_feature.to_s) if ex.pred_feat
+# post "/test_validation",:select=>"6d" #,:report=>"yes,please"
+# puts last_response.body
- post '/create_validation', { :test_dataset_uri => ex.test_data, :model_uri => ex.model, :prediction_dataset_uri=> ex.pred_data}
- puts last_response.body
+ #run_test("6a")
+
+ #puts Nightly.build_nightly("6")
- val_uri = wait_for_task(last_response.body)
- puts val_uri
-
- get val_uri
- puts last_response.body
-
- #puts Lib::OTPredictions.new( ex.classification, ex.test_data, ex.act_feat, ex.pred_data, ex.pred_feat ).compute_stats.each{|key,value| puts key.to_s+" => "+value.to_s }
+ #prepare_examples
+ do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE
end
-
- def validate_split(ex)
- begin
-
-# model = OpenTox::Model::PredictionModel.find("http://ot.model.de/66")
-# puts model.predicted_variables
-# exit
-
- #data_uri = upload_data(WS_DATA, FILE)
- #data_uri = "http://ot.dataset.de/199" #bbrc
- #data_uri = "http://ot.dataset.de/67" #hamster
- #puts data_uri
-
- #exit
-
- #data_uri=WS_DATA+"/"+DATA
-# post '/training_test_split', { :dataset_uri => data_uri, :algorithm_uri => WS_CLASS_ALG, :prediction_feature => FEATURE_URI,
-# :algorithm_params => "feature_generation_uri="+WS_FEATURE_ALG, :split_ratio=>0.75, :random_seed=>6}
- post '/training_test_split', { :dataset_uri => ex.orig_data, :algorithm_uri => ex.alg, :prediction_feature => ex.act_feat,
- :algorithm_params => ex.alg_params, :split_ratio=>0.75, :random_seed=>6 }
-
- puts last_response.body
-
- val_uri = wait_for_task(last_response.body)
- puts val_uri
-
- get val_uri
- puts last_response.body
- #verify_validation
- ensure
- #delete_resources
- end
+ def app
+ Sinatra::Application
end
-
- def verify_validation(val_yaml)
-
- puts val_yaml
- val = YAML.load(val_yaml)
-
- puts val.inspect
- assert_integer val["num_instances".to_sym],0,1000
- num_instances = val["num_instances".to_sym].to_i
-
- assert_integer val["num_unpredicted".to_sym],0,num_instances
- num_unpredicted = val["num_unpredicted".to_sym].to_i
- assert_float val["percent_unpredicted".to_sym],0,100
- assert_float_equal(val["percent_unpredicted".to_sym].to_f,100*num_unpredicted/num_instances.to_f,"percent_unpredicted")
-
- assert_integer val["num_without_class".to_sym],0,num_instances
- num_without_class = val["num_without_class".to_sym].to_i
- assert_float val["percent_without_class".to_sym],0,100
- assert_float_equal(val["percent_without_class".to_sym].to_f,100*num_without_class/num_instances.to_f,"percent_without_class")
-
- class_stats = val["classification_statistics".to_sym]
- if class_stats
- class_value_stats = class_stats["class_value_statistics".to_sym]
- class_values = []
- class_value_stats.each do |cvs|
- class_values << cvs["class_value".to_sym]
+ def run_test(select)
+ $test_case = self
+ validationExamples = ValidationExamples.select(select)
+ validationExamples.each do |vv|
+ vv.each do |v|
+ ex = v.new
+ ex.upload_files
+ ex.check_requirements
+ ex.validate
+ ex.report
end
- puts class_values.inspect
-
- confusion_matrix = class_stats["confusion_matrix".to_sym]
- confusion_matrix_cells = confusion_matrix["confusion_matrix_cell".to_sym]
- predictions = 0
- confusion_matrix_cells.each do |confusion_matrix_cell|
- predictions += confusion_matrix_cell["confusion_matrix_value".to_sym].to_i
- end
- assert_int_equal(predictions, num_instances-num_unpredicted)
- else
- regr_stats = val["regression_statistics".to_sym]
- assert regr_stats!=nil
end
end
- def assert_int_equal(val1,val2,msg_suffix=nil)
- assert(val1==val2,msg_suffix.to_s+" not equal: "+val1.to_s+" != "+val2.to_s)
- end
-
- def assert_float_equal(val1,val2,msg_suffix=nil,epsilon=0.0001)
- assert((val1-val2).abs<epsilon,msg_suffix.to_s+" not equal: "+val1.to_s+" != "+val2.to_s+", diff:"+(val1-val2).abs.to_s)
- end
-
- def assert_integer(string_val, min=nil, max=nil)
- assert string_val.to_i.to_s==string_val.to_s, string_val.to_s+" not an integer"
- assert string_val.to_i>=min if min!=nil
- assert string_val.to_i<=max if max!=nil
- end
-
- def assert_float(string_val, min=nil, max=nil)
- assert( string_val.to_f.to_s==string_val.to_s || (string_val.to_f.to_s==(string_val.to_s+".0")),
- string_val.to_s+" not a float (!="+string_val.to_f.to_s+")")
- assert string_val.to_f>=min if min!=nil
- assert string_val.to_f<=max if max!=nil
- end
-
-# def test_nothing
-#
-# #puts "testing nothing"
-#
-# #get '/'
-#
-# #get '/crossvalidation/loo'
-# #get '/training_test_split'
-#
-# #get '/1',nil,'HTTP_ACCEPT' => "application/rdf+xml"
-# #get '/1',nil,'HTTP_ACCEPT' => "text/x-yaml"
-#
-#
-# #get '/crossvalidation/1',nil,'HTTP_ACCEPT' => "application/rdf+xml"
-# #get '/crossvalidation/1/statistics',nil,'HTTP_ACCEPT' => "text/x-yaml"
-#
-# #puts last_response.body
-#
-# #get '/2'
-# #verify_validation(last_response.body)
-#
-# end
-
-# private
-# def verify_validation (delete=true)
-#
-# puts "validation: "+last_response.body
-# assert last_response.ok?
-# validation_id = last_response.body.split("/")[-1]
-#
-# puts "uri: "+last_response.body
-# puts "id:"+validation_id
-# add_resource("/"+validation_id) if delete
-#
-# #get '/'+validation_id,nil,'HTTP_ACCEPT' => "application/rdf+xml"
-# get '/'+validation_id,nil,'HTTP_ACCEPT' => "text/x-yaml"
-# puts last_response.body
-# assert last_response.ok? || last_response.status==202
-#
-## ["test_dataset_uri", "model_uri", "prediction_dataset_uri"].each do |t|
-## get '/'+validation_id+'/'+t
-## puts ""+t+": "+last_response.body
-## assert last_response.ok?
-##
-## content = ext("curl "+last_response.body)
-## content = content.split("\n")[0,10].join("\n")+"\n...\n" if content.count("\n")>10
-## puts content
-## end
-# end
-
def prepare_examples
get '/prepare_examples'
end
@@ -642,4 +52,5 @@ def ex_tum2
def do_test_examples # USES CURL, DO NOT FORGET TO RESTART
get '/test_examples'
end
+
end
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-02 02:09:16 +0000