diff options
-rw-r--r-- | application.rb | 1 | ||||
-rw-r--r-- | data/hamster_carcinogenicity.test.yaml | 119 | ||||
-rw-r--r-- | data/hamster_carcinogenicity.train.yaml | 178 | ||||
-rw-r--r-- | nightly/nightly.rb | 598 | ||||
-rw-r--r-- | nightly/nightly_application.rb | 4 | ||||
-rw-r--r-- | report/report_test.rb | 10 | ||||
-rw-r--r-- | test/test_application.rb | 17 | ||||
-rw-r--r-- | test/test_examples.rb | 214 | ||||
-rw-r--r-- | test/test_examples_util.rb | 194 | ||||
-rw-r--r-- | validation/validation_test.rb | 631 |
10 files changed, 831 insertions, 1135 deletions
diff --git a/application.rb b/application.rb index ca2a7ac..236280c 100644 --- a/application.rb +++ b/application.rb @@ -29,6 +29,7 @@ get '/test_examples/?' do Example.test_examples end +require "test/test_application.rb" require "nightly/nightly_application.rb" # order is important, first add example methods and reports, than validation diff --git a/data/hamster_carcinogenicity.test.yaml b/data/hamster_carcinogenicity.test.yaml new file mode 100644 index 0000000..32d305a --- /dev/null +++ b/data/hamster_carcinogenicity.test.yaml @@ -0,0 +1,119 @@ +--- !ruby/object:OpenTox::Dataset +compounds: +- http://localhost/compound/InChI=1S/C9H6O2/c10-9-6-5-7-3-1-2-4-8(7)11-9/h1-6H +- http://localhost/compound/InChI=1S/2C2H4O2.4H2O.3Pb/c2*1-2(3)4;;;;;;;/h2*1H3,(H,3,4);4*1H2;;;/q;;;;;;3*+2/p-6 +- http://localhost/compound/InChI=1S/C10H12ClNO2/c1-7(2)14-10(13)12-9-5-3-4-8(11)6-9/h3-7H,1-2H3,(H,12,13) +- http://localhost/compound/InChI=1S/C8H6N4O4S/c13-4-9-11-8-10-5(3-17-8)6-1-2-7(16-6)12(14)15/h1-4H,(H,9,13)(H,10,11) +- http://localhost/compound/InChI=1S/C4H7N3O3/c1-3(8)2-7(6-10)4(5)9/h2H2,1H3,(H2,5,9) +- http://localhost/compound/InChI=1S/C5H10N2O/c8-6-7-4-2-1-3-5-7/h1-5H2 +- http://localhost/compound/InChI=1S/C15H13NO/c1-10(17)16-13-6-7-15-12(9-13)8-11-4-2-3-5-14(11)15/h2-7,9H,8H2,1H3,(H,16,17) +- http://localhost/compound/InChI=1S/C4H6N2O3/c1-3-2-6(5-8)4(7)9-3/h3H,2H2,1H3 +- http://localhost/compound/InChI=1S/CH6N2/c1-3-2/h3H,2H2,1H3 +- http://localhost/compound/InChI=1S/C2H6N2O/c1-4(3)2-5/h2H,3H2,1H3 +- http://localhost/compound/InChI=1S/C6H12N2O2/c1-5-3-8(7-9)4-6(2)10-5/h5-6H,3-4H2,1-2H3 +- http://localhost/compound/InChI=1S/C9H11N3O2/c10-9(13)12(11-14)7-6-8-4-2-1-3-5-8/h1-5H,6-7H2,(H2,10,13) +- http://localhost/compound/InChI=1S/C5H6N2OS/c1-3-2-4(8)7-5(9)6-3/h2H,1H3,(H2,6,7,8,9) +- http://localhost/compound/InChI=1S/C5H11N3O3/c1-2-8(7-11)5(10)6-3-4-9/h9H,2-4H2,1H3,(H,6,10) +- http://localhost/compound/InChI=1S/C3H7NO2/c1-2-6-3(4)5/h2H2,1H3,(H2,4,5) +- http://localhost/compound/InChI=1S/C14H19N3S.ClH/c1-16(2)9-10-17(12-13-6-5-11-18-13)14-7-3-4-8-15-14;/h3-8,11H,9-10,12H2,1-2H3;1H +- http://localhost/compound/InChI=1S/C5H4O2/c6-4-5-2-1-3-7-5/h1-4H +- http://localhost/compound/InChI=1S/C14H9Cl5/c15-11-5-1-9(2-6-11)13(14(17,18)19)10-3-7-12(16)8-4-10/h1-8,13H +- http://localhost/compound/InChI=1S/H4N2.H2O4S/c1-2;1-5(2,3)4/h1-2H2;(H2,1,2,3,4) +- http://localhost/compound/InChI=1S/C4H8N2O3/c1-3-9-4(7)6(2)5-8/h3H2,1-2H3 +- http://localhost/compound/InChI=1S/C3H6O2/c4-1-3-2-5-3/h3-4H,1-2H2 +- http://localhost/compound/InChI=1S/C19H17N3.ClH/c20-16-7-1-13(2-8-16)19(14-3-9-17(21)10-4-14)15-5-11-18(22)12-6-15;/h1-12,20H,21-22H2;1H +- http://localhost/compound/InChI=1S/C9H7N3O4S/c1-5(13)10-9-11-6(4-17-9)7-2-3-8(16-7)12(14)15/h2-4H,1H3,(H,10,11,13) +- http://localhost/compound/InChI=1S/C6H12N4O2/c1-5-3-9(7-11)4-6(2)10(5)8-12/h5-6H,3-4H2,1-2H3 +- http://localhost/compound/InChI=1S/C15H10O7.2H2O/c16-7-4-10(19)12-11(5-7)22-15(14(21)13(12)20)6-1-2-8(17)9(18)3-6;;/h1-5,16-19,21H;2*1H2 +- http://localhost/compound/InChI=1S/C6H7N3O/c7-9-6(10)5-1-3-8-4-2-5/h1-4H,7H2,(H,9,10) +- http://localhost/compound/InChI=1S/C6H11N3O3/c1-3-9(8-12)6(11)7-4-5(2)10/h3-4H2,1-2H3,(H,7,11) +- http://localhost/compound/InChI=1S/C5H8O2/c1-4(2)5(6)7-3/h1H2,2-3H3 +creator: http://example.org/training_test_split +data: + http://localhost/compound/InChI=1S/C9H11N3O2/c10-9(13)12(11-14)7-6-8-4-2-1-3-5-8/h1-5H,6-7H2,(H2,10,13): + - {} + + http://localhost/compound/InChI=1S/C9H6O2/c10-9-6-5-7-3-1-2-4-8(7)11-9/h1-6H: + - {} + + http://localhost/compound/InChI=1S/C3H6O2/c4-1-3-2-5-3/h3-4H,1-2H2: + - {} + + http://localhost/compound/InChI=1S/C4H8N2O3/c1-3-9-4(7)6(2)5-8/h3H2,1-2H3: + - {} + + http://localhost/compound/InChI=1S/C5H6N2OS/c1-3-2-4(8)7-5(9)6-3/h2H,1H3,(H2,6,7,8,9): + - {} + + http://localhost/compound/InChI=1S/C6H12N2O2/c1-5-3-8(7-9)4-6(2)10-5/h5-6H,3-4H2,1-2H3: + - {} + + http://localhost/compound/InChI=1S/C10H12ClNO2/c1-7(2)14-10(13)12-9-5-3-4-8(11)6-9/h3-7H,1-2H3,(H,12,13): + - {} + + http://localhost/compound/InChI=1S/C9H7N3O4S/c1-5(13)10-9-11-6(4-17-9)7-2-3-8(16-7)12(14)15/h2-4H,1H3,(H,10,11,13): + - {} + + http://localhost/compound/InChI=1S/C6H11N3O3/c1-3-9(8-12)6(11)7-4-5(2)10/h3-4H2,1-2H3,(H,7,11): + - {} + + http://localhost/compound/InChI=1S/C5H4O2/c6-4-5-2-1-3-7-5/h1-4H: + - {} + + http://localhost/compound/InChI=1S/C8H6N4O4S/c13-4-9-11-8-10-5(3-17-8)6-1-2-7(16-6)12(14)15/h1-4H,(H,9,13)(H,10,11): + - {} + + http://localhost/compound/InChI=1S/C14H9Cl5/c15-11-5-1-9(2-6-11)13(14(17,18)19)10-3-7-12(16)8-4-10/h1-8,13H: + - {} + + http://localhost/compound/InChI=1S/C3H7NO2/c1-2-6-3(4)5/h2H2,1H3,(H2,4,5): + - {} + + http://localhost/compound/InChI=1S/C15H13NO/c1-10(17)16-13-6-7-15-12(9-13)8-11-4-2-3-5-14(11)15/h2-7,9H,8H2,1H3,(H,16,17): + - {} + + http://localhost/compound/InChI=1S/C5H10N2O/c8-6-7-4-2-1-3-5-7/h1-5H2: + - {} + + http://localhost/compound/InChI=1S/C19H17N3.ClH/c20-16-7-1-13(2-8-16)19(14-3-9-17(21)10-4-14)15-5-11-18(22)12-6-15;/h1-12,20H,21-22H2;1H: + - {} + + http://localhost/compound/InChI=1S/H4N2.H2O4S/c1-2;1-5(2,3)4/h1-2H2;(H2,1,2,3,4): + - {} + + http://localhost/compound/InChI=1S/C6H7N3O/c7-9-6(10)5-1-3-8-4-2-5/h1-4H,7H2,(H,9,10): + - {} + + http://localhost/compound/InChI=1S/CH6N2/c1-3-2/h3H,2H2,1H3: + - {} + + http://localhost/compound/InChI=1S/C6H12N4O2/c1-5-3-9(7-11)4-6(2)10(5)8-12/h5-6H,3-4H2,1-2H3: + - {} + + http://localhost/compound/InChI=1S/C2H6N2O/c1-4(3)2-5/h2H,3H2,1H3: + - {} + + http://localhost/compound/InChI=1S/C4H6N2O3/c1-3-2-6(5-8)4(7)9-3/h3H,2H2,1H3: + - {} + + http://localhost/compound/InChI=1S/C14H19N3S.ClH/c1-16(2)9-10-17(12-13-6-5-11-18-13)14-7-3-4-8-15-14;/h3-8,11H,9-10,12H2,1-2H3;1H: + - {} + + http://localhost/compound/InChI=1S/C4H7N3O3/c1-3(8)2-7(6-10)4(5)9/h2H2,1H3,(H2,5,9): + - {} + + http://localhost/compound/InChI=1S/2C2H4O2.4H2O.3Pb/c2*1-2(3)4;;;;;;;/h2*1H3,(H,3,4);4*1H2;;;/q;;;;;;3*+2/p-6: + - {} + + http://localhost/compound/InChI=1S/C5H8O2/c1-4(2)5(6)7-3/h1H2,2-3H3: + - {} + + http://localhost/compound/InChI=1S/C15H10O7.2H2O/c16-7-4-10(19)12-11(5-7)22-15(14(21)13(12)20)6-1-2-8(17)9(18)3-6;;/h1-5,16-19,21H;2*1H2: + - {} + + http://localhost/compound/InChI=1S/C5H11N3O3/c1-2-8(7-11)5(10)6-3-4-9/h9H,2-4H2,1H3,(H,6,10): + - {} + +features: [] + +title: Test dataset split of Hamster Carcinogenicity (DSSTOX/CPDB) diff --git a/data/hamster_carcinogenicity.train.yaml b/data/hamster_carcinogenicity.train.yaml new file mode 100644 index 0000000..fd1544f --- /dev/null +++ b/data/hamster_carcinogenicity.train.yaml @@ -0,0 +1,178 @@ +--- !ruby/object:OpenTox::Dataset +compounds: +- http://localhost/compound/InChI=1S/C11H8N2O5/c12-11(14)8(9-2-1-5-17-9)6-7-3-4-10(18-7)13(15)16/h1-6H,(H2,12,14) +- http://localhost/compound/InChI=1S/C20H22N8O5/c1-28(9-11-8-23-17-15(24-11)16(21)26-20(22)27-17)12-4-2-10(3-5-12)18(31)25-13(19(32)33)6-7-14(29)30/h2-5,8,13H,6-7,9H2,1H3,(H,25,31)(H,29,30)(H,32,33)(H4,21,22,23,26,27)/t13-/m0/s1 +- http://localhost/compound/InChI=1S/C6H12N2O4/c1-5(10)2-8(7-12)3-6(11)4-9/h6,9,11H,2-4H2,1H3 +- http://localhost/compound/InChI=1S/C6H10N2O/c1-3-5-8(7-9)6-4-2/h3-4H,1-2,5-6H2 +- http://localhost/compound/InChI=1S/C6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9 +- http://localhost/compound/InChI=1S/C12H12N2O3/c1-2-12(8-6-4-3-5-7-8)9(15)13-11(17)14-10(12)16/h3-7H,2H2,1H3,(H2,13,14,15,16,17) +- http://localhost/compound/InChI=1S/C4H10N2O3/c1-6(5-9)2-4(8)3-7/h4,7-8H,2-3H2,1H3 +- http://localhost/compound/InChI=1S/C17H17ClO6/c1-8-5-9(19)6-12(23-4)17(8)16(20)13-10(21-2)7-11(22-3)14(18)15(13)24-17/h6-8H,5H2,1-4H3/t8-,17?/m1/s1 +- http://localhost/compound/InChI=1S/Cd.2ClH/h;2*1H/q+2;;/p-2 +- http://localhost/compound/InChI=1S/C15H13NO2/c1-10(17)16(18)13-6-7-15-12(9-13)8-11-4-2-3-5-14(11)15/h2-7,9,18H,8H2,1H3 +- http://localhost/compound/InChI=1S/C4H8N2O/c7-5-6-3-1-2-4-6/h1-4H2 +- http://localhost/compound/InChI=1S/C20H22O3/c1-20(2,19(21)22)23-16-12-10-15(11-13-16)18-9-5-7-14-6-3-4-8-17(14)18/h3-4,6,8,10-13,18H,5,7,9H2,1-2H3,(H,21,22) +- http://localhost/compound/InChI=1S/C6H14N2O4/c1-5(10)2-8(7-12)3-6(11)4-9/h5-6,9-11H,2-4H2,1H3 +- http://localhost/compound/InChI=1S/C27H30O16/c1-8-17(32)20(35)22(37)26(40-8)39-7-15-18(33)21(36)23(38)27(42-15)43-25-19(34)16-13(31)5-10(28)6-14(16)41-24(25)9-2-3-11(29)12(30)4-9/h2-6,8,15,17-18,20-23,26-33,35-38H,7H2,1H3/t8-,15+,17-,18+,20+,21-,22+,23+,26+,27?/m0/s1 +- http://localhost/compound/InChI=1S/C2HCl3/c3-1-2(4)5/h1H +- http://localhost/compound/InChI=1S/C2H8N2.2ClH/c1-3-4-2;;/h3-4H,1-2H3;2*1H +- http://localhost/compound/InChI=1S/C9H11N3O/c13-11-12-6-2-4-9(12)8-3-1-5-10-7-8/h1,3,5,7,9H,2,4,6H2 +- http://localhost/compound/InChI=1S/BrHO3.K/c2-1(3)4;/h(H,2,3,4);/q;+1/p-1 +- http://localhost/compound/InChI=1S/C6H5N2.BF4/c7-8-6-4-2-1-3-5-6;2-1(3,4)5/h1-5H;/q+1;-1 +- http://localhost/compound/InChI=1S/H4N2/c1-2/h1-2H2 +- http://localhost/compound/InChI=1S/Cd.H2O4S/c;1-5(2,3)4/h;(H2,1,2,3,4)/q+2;/p-2 +- http://localhost/compound/InChI=1S/C2H6O/c1-2-3/h3H,2H2,1H3 +- http://localhost/compound/InChI=1S/C14H8Cl4/c15-11-5-1-9(2-6-11)13(14(17)18)10-3-7-12(16)8-4-10/h1-8H +- http://localhost/compound/InChI=1S/C5H10N2O3/c1-5(9)4-7(6-10)2-3-8/h8H,2-4H2,1H3 +- http://localhost/compound/InChI=1S/C7H15N3O/c1-6-4-10(8-11)5-7(2)9(6)3/h6-7H,4-5H2,1-3H3 +- http://localhost/compound/InChI=1S/BF4.Na/c2-1(3,4)5;/q-1;+1 +- http://localhost/compound/InChI=1S/C10H13N3O2/c1-13(12-15)7-3-5-10(14)9-4-2-6-11-8-9/h2,4,6,8H,3,5,7H2,1H3 +- http://localhost/compound/InChI=1S/C12H9NO2/c14-13(15)11-7-6-9-5-4-8-2-1-3-10(11)12(8)9/h1-3,6-7H,4-5H2 +- http://localhost/compound/InChI=1S/C8H5N3O4S/c12-4-9-8-10-5(3-16-8)6-1-2-7(15-6)11(13)14/h1-4H,(H,9,10,12) +- http://localhost/compound/InChI=1S/C2H4N4/c3-2-4-1-5-6-2/h1H,(H3,3,4,5,6) +- http://localhost/compound/InChI=1S/CH2O/c1-2/h1H2 +- http://localhost/compound/InChI=1S/C10H13NO2/c1-8(2)13-10(12)11-9-6-4-3-5-7-9/h3-8H,1-2H3,(H,11,12) +- http://localhost/compound/InChI=1S/C14H14ClN3O2S/c1-8-4-3-5-10(9(8)2)16-12-6-11(15)17-14(18-12)21-7-13(19)20/h3-6H,7H2,1-2H3,(H,19,20)(H,16,17,18) +- http://localhost/compound/InChI=1S/C5H13N3O/c1-7(2)4-5-8(3)6-9/h4-5H2,1-3H3 +- http://localhost/compound/InChI=1S/C3H6N2O/c6-4-5-2-1-3-5/h1-3H2 +- http://localhost/compound/InChI=1S/C2H5ClO/c1-4-2-3/h2H2,1H3 +- http://localhost/compound/InChI=1S/C2H3Cl/c1-2-3/h2H,1H2 +- http://localhost/compound/InChI=1S/C6H10ClN3O3/c1-5(11)4-10(9-13)6(12)8-3-2-7/h2-4H2,1H3,(H,8,12) +- http://localhost/compound/InChI=1S/C9H9NS/c11-8-10-7-6-9-4-2-1-3-5-9/h1-5H,6-7H2 +- http://localhost/compound/InChI=1S/C2H4O/c1-2-3/h2H,1H3 +- http://localhost/compound/InChI=1S/C3H6ClNO/c1-5(2)3(4)6/h1-2H3 +- http://localhost/compound/InChI=1S/C17H17ClO3/c1-17(2,16(19)20)21-11-12-3-5-13(6-4-12)14-7-9-15(18)10-8-14/h3-10H,11H2,1-2H3,(H,19,20) +- http://localhost/compound/InChI=1S/C2H8N2O/c3-4-1-2-5/h4-5H,1-3H2 +- http://localhost/compound/InChI=1S/HNO2.Na/c2-1-3;/h(H,2,3);/q;+1/p-1 +- http://localhost/compound/InChI=1S/C6H10N2O2/c1-3-4-8(7-10)5-6(2)9/h3H,1,4-5H2,2H3 +- http://localhost/compound/InChI=1S/C7H6O4/c8-5-2-1-4(7(10)11)3-6(5)9/h1-3,8-9H,(H,10,11) +- http://localhost/compound/InChI=1S/C6H5NO2/c8-6(9)5-1-3-7-4-2-5/h1-4H,(H,8,9) +- http://localhost/compound/InChI=1S/C16H13N/c1-2-8-15(9-3-1)17-16-11-10-13-6-4-5-7-14(13)12-16/h1-12,17H +- http://localhost/compound/InChI=1S/C4H5Cl/c1-3-4(2)5/h3H,1-2H2 +- http://localhost/compound/InChI=1S/C2H8N2/c1-4(2)3/h3H2,1-2H3 +- http://localhost/compound/InChI=1S/C4H8N2O2/c7-5-6-1-3-8-4-2-6/h1-4H2 +- http://localhost/compound/InChI=1S/C20H19N3.ClH/c1-13-12-16(6-11-19(13)23)20(14-2-7-17(21)8-3-14)15-4-9-18(22)10-5-15;/h2-12,21H,22-23H2,1H3;1H +- http://localhost/compound/InChI=1S/C9H11N3O2/c13-10-12-6-2-4-9(12)8-3-1-5-11(14)7-8/h1,3,5,7,9H,2,4,6H2 +- http://localhost/compound/InChI=1S/C3H6N2O2/c6-4-5-1-2-7-3-5/h1-3H2 +- http://localhost/compound/InChI=1S/C19H24N2O2/c22-18-13-20(19(23)15-7-2-1-3-8-15)12-17-16-9-5-4-6-14(16)10-11-21(17)18/h4-6,9,15,17H,1-3,7-8,10-13H2 +- http://localhost/compound/InChI=1S/C12H8Cl6O/c13-8-9(14)11(16)5-3-1-2(6-7(3)19-6)4(5)10(8,15)12(11,17)18/h2-7H,1H2 +- http://localhost/compound/InChI=1S/C5H12N2O4/c8-2-1-7(6-11)3-5(10)4-9/h5,8-10H,1-4H2 +creator: http://example.org/training_test_split +data: + http://localhost/compound/InChI=1S/C7H6O4/c8-5-2-1-4(7(10)11)3-6(5)9/h1-3,8-9H,(H,10,11): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C3H6N2O/c6-4-5-2-1-3-5/h1-3H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C10H13N3O2/c1-13(12-15)7-3-5-10(14)9-4-2-6-11-8-9/h2,4,6,8H,3,5,7H2,1H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C2H6O/c1-2-3/h3H,2H2,1H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C6H14N2O4/c1-5(10)2-8(7-12)3-6(11)4-9/h5-6,9-11H,2-4H2,1H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C4H8N2O/c7-5-6-3-1-2-4-6/h1-4H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/Cd.2ClH/h;2*1H/q+2;;/p-2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C17H17ClO6/c1-8-5-9(19)6-12(23-4)17(8)16(20)13-10(21-2)7-11(22-3)14(18)15(13)24-17/h6-8H,5H2,1-4H3/t8-,17?/m1/s1: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C4H10N2O3/c1-6(5-9)2-4(8)3-7/h4,7-8H,2-3H2,1H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C20H22N8O5/c1-28(9-11-8-23-17-15(24-11)16(21)26-20(22)27-17)12-4-2-10(3-5-12)18(31)25-13(19(32)33)6-7-14(29)30/h2-5,8,13H,6-7,9H2,1H3,(H,25,31)(H,29,30)(H,32,33)(H4,21,22,23,26,27)/t13-/m0/s1: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C2H8N2O/c3-4-1-2-5/h4-5H,1-3H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C17H17ClO3/c1-17(2,16(19)20)21-11-12-3-5-13(6-4-12)14-7-9-15(18)10-8-14/h3-10H,11H2,1-2H3,(H,19,20): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C2H4O/c1-2-3/h2H,1H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C2H5ClO/c1-4-2-3/h2H2,1H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C8H5N3O4S/c12-4-9-8-10-5(3-16-8)6-1-2-7(15-6)11(13)14/h1-4H,(H,9,10,12): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C9H11N3O/c13-11-12-6-2-4-9(12)8-3-1-5-10-7-8/h1,3,5,7,9H,2,4,6H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C20H22O3/c1-20(2,19(21)22)23-16-12-10-15(11-13-16)18-9-5-7-14-6-3-4-8-17(14)18/h3-4,6,8,10-13,18H,5,7,9H2,1-2H3,(H,21,22): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C11H8N2O5/c12-11(14)8(9-2-1-5-17-9)6-7-3-4-10(18-7)13(15)16/h1-6H,(H2,12,14): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C20H19N3.ClH/c1-13-12-16(6-11-19(13)23)20(14-2-7-17(21)8-3-14)15-4-9-18(22)10-5-15;/h2-12,21H,22-23H2,1H3;1H: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C16H13N/c1-2-8-15(9-3-1)17-16-11-10-13-6-4-5-7-14(13)12-16/h1-12,17H: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C12H9NO2/c14-13(15)11-7-6-9-5-4-8-2-1-3-10(11)12(8)9/h1-3,6-7H,4-5H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C15H13NO2/c1-10(17)16(18)13-6-7-15-12(9-13)8-11-4-2-3-5-14(11)15/h2-7,9,18H,8H2,1H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C4H8N2O2/c7-5-6-1-3-8-4-2-6/h1-4H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C9H9NS/c11-8-10-7-6-9-4-2-1-3-5-9/h1-5H,6-7H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/BF4.Na/c2-1(3,4)5;/q-1;+1: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C7H15N3O/c1-6-4-10(8-11)5-7(2)9(6)3/h6-7H,4-5H2,1-3H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C5H10N2O3/c1-5(9)4-7(6-10)2-3-8/h8H,2-4H2,1H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/Cd.H2O4S/c;1-5(2,3)4/h;(H2,1,2,3,4)/q+2;/p-2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/BrHO3.K/c2-1(3)4;/h(H,2,3,4);/q;+1/p-1: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C3H6N2O2/c6-4-5-1-2-7-3-5/h1-3H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C4H5Cl/c1-3-4(2)5/h3H,1-2H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C6H10ClN3O3/c1-5(11)4-10(9-13)6(12)8-3-2-7/h2-4H2,1H3,(H,8,12): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C12H12N2O3/c1-2-12(8-6-4-3-5-7-8)9(15)13-11(17)14-10(12)16/h3-7H,2H2,1H3,(H2,13,14,15,16,17): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C6H12N2O4/c1-5(10)2-8(7-12)3-6(11)4-9/h6,9,11H,2-4H2,1H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C12H8Cl6O/c13-8-9(14)11(16)5-3-1-2(6-7(3)19-6)4(5)10(8,15)12(11,17)18/h2-7H,1H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C19H24N2O2/c22-18-13-20(19(23)15-7-2-1-3-8-15)12-17-16-9-5-4-6-14(16)10-11-21(17)18/h4-6,9,15,17H,1-3,7-8,10-13H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C6H10N2O2/c1-3-4-8(7-10)5-6(2)9/h3H,1,4-5H2,2H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C3H6ClNO/c1-5(2)3(4)6/h1-2H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C10H13NO2/c1-8(2)13-10(12)11-9-6-4-3-5-7-9/h3-8H,1-2H3,(H,11,12): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C2H4N4/c3-2-4-1-5-6-2/h1H,(H3,3,4,5,6): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C14H8Cl4/c15-11-5-1-9(2-6-11)13(14(17)18)10-3-7-12(16)8-4-10/h1-8H: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/H4N2/c1-2/h1-2H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C9H11N3O2/c13-10-12-6-2-4-9(12)8-3-1-5-11(14)7-8/h1,3,5,7,9H,2,4,6H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C5H13N3O/c1-7(2)4-5-8(3)6-9/h4-5H2,1-3H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/HNO2.Na/c2-1-3;/h(H,2,3);/q;+1/p-1: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C2HCl3/c3-1-2(4)5/h1H: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C5H12N2O4/c8-2-1-7(6-11)3-5(10)4-9/h5,8-10H,1-4H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C2H8N2/c1-4(2)3/h3H2,1-2H3: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C6H5NO2/c8-6(9)5-1-3-7-4-2-5/h1-4H,(H,8,9): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/CH2O/c1-2/h1H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C6H5N2.BF4/c7-8-6-4-2-1-3-5-6;2-1(3,4)5/h1-5H;/q+1;-1: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C27H30O16/c1-8-17(32)20(35)22(37)26(40-8)39-7-15-18(33)21(36)23(38)27(42-15)43-25-19(34)16-13(31)5-10(28)6-14(16)41-24(25)9-2-3-11(29)12(30)4-9/h2-6,8,15,17-18,20-23,26-33,35-38H,7H2,1H3/t8-,15+,17-,18+,20+,21-,22+,23+,26+,27?/m0/s1: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C6H10N2O/c1-3-5-8(7-9)6-4-2/h3-4H,1-2,5-6H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C2H3Cl/c1-2-3/h2H,1H2: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true + http://localhost/compound/InChI=1S/C14H14ClN3O2S/c1-8-4-3-5-10(9(8)2)16-12-6-11(15)17-14(18-12)21-7-13(19)20/h3-6H,7H2,1-2H3,(H,19,20)(H,16,17,18): + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): false + http://localhost/compound/InChI=1S/C2H8N2.2ClH/c1-3-4-2;;/h3-4H,1-2H3;2*1H: + - http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB): true +features: +- http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB) +title: Training dataset split of Hamster Carcinogenicity (DSSTOX/CPDB) diff --git a/nightly/nightly.rb b/nightly/nightly.rb index dc06ae3..594d4ee 100644 --- a/nightly/nightly.rb +++ b/nightly/nightly.rb @@ -1,4 +1,5 @@ +require 'test/test_examples.rb' class Nightly @@ -15,18 +16,15 @@ class Nightly end end - def self.build_nightly(dry_run=false) + def self.build_nightly(select=nil, dry_run=false) + + validationExamples = ValidationExamples.select(select) + return "please \"select\" validation examples:\n"+ValidationExamples.list if validationExamples.size==0 + task_uri = OpenTox::Task.as_task() do LOGGER.info("Building nightly report") - benchmarks = [ HamsterTrainingTestBenchmark.new, - #HamsterCrossvalidationBenchmark.new, - #MiniRegressionBenchmark.new, - #CacoModelsRegressionBenchmark.new, - #CacoModelsRegressionBenchmark2.new, - #CacoAlgsRegressionBenchmark.new, - #FatheadRegressionBenchmark.new, - ] + benchmarks = validationExamples.collect{ |e| ValidationBenchmark.new(e) } running = [] report = Reports::XMLReport.new("Nightly Validation", Time.now.strftime("Created at %m.%d.%Y - %H:%M")) @@ -60,15 +58,14 @@ class Nightly section = report.add_section(report.get_root_element, b.title) section_info = report.add_section(section, "Info") - b.info.each{|i| report.add_paragraph(section_info,i)} info_table = b.info_table - report.add_table(section_info, b.info_table_title, info_table) if info_table + report.add_table(section_info, "Validation info", info_table) if info_table section_results = report.add_section(section, "Results") - report.add_table(section_results, b.result_table_title, b.result_table) + report.add_table(section_results, "Valdation results", b.result_table) if (b.comparison_report) - report.add_table(section_results, b.comparison_report_title, [[b.comparison_report]], false) + report.add_table(section_results, "Validation comparison report", [[b.comparison_report]], false) end section_errors = report.add_section(section, "Errors") @@ -96,7 +93,7 @@ class Nightly else LOGGER.info("Nightly report completed - DRY RUN, no report creation") end - return "Nightly report completed" + benchmarks.collect{|b| b.uris}.join(",") end if defined?(halt) halt 202,task_uri @@ -105,120 +102,102 @@ class Nightly end end - class AbstractBenchmark - - def info_table_title - return title + class ValidationBenchmark + + attr_accessor :errors, :comparison_report + + def comparable + if @comp == nil + @comp = @validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri + end + @comp end - def info_table - return nil + def uris + @validation_examples.collect{|v| v.validation_uri}.join(",") + end + + def initialize(validationExamples) + @validation_examples = [] + validationExamples.each do |v| + example = v.new + @validation_examples << example + end end - def result_table_title - return "Validation results" + def title + @validation_examples.collect{|e| e.title}.join(" + ") end def result_table - raise "no comparables" unless @comparables - raise "no validations" unless @validations - raise "no reports" unless @reports t = [] - row = [comparable_nice_name, "validation", "report"] + row = [comparable.to_s, "validation", "report"] t << row - (0..@comparables.size-1).each do |i| - row = [ @comparables[i], @validations[i], @reports[i] ] + @validation_examples.each do |e| + row = [ e.send(comparable), + (e.validation_error!=nil ? "error, see below" : e.validation_uri), + (e.report_error!=nil ? "error, see below" : e.report_uri) ] t << row end t end - - def comparison_report_title - "algorithm comparison report" - end - - def comparison_report - @comparison_report - end - - end - - class ValidationBenchmark < AbstractBenchmark def info_table - raise "no comparables" unless @comparables t = [] t << ["param", "uri"] - params.each do |k,v| - t << [k.to_s, v.to_s] - end - count = 1 - @comparables.each do |alg| - t << [comparable_nice_name+" ["+count.to_s+"]", alg] - count += 1 + + (@validation_examples[0].params+@validation_examples[0].opt_params).each do |p| + map = {} + @validation_examples.each{|e| map[e.send(p).to_s]=nil } + + if map.size==1 && map.keys[0].size==0 + #omit + elsif map.size==1 #values equal + t << [p.to_s, map.keys[0]] + else + count = 1 + @validation_examples.each do |e| + t << [p.to_s+" ["+count.to_s+"]", e.send(p)] + count += 1 + end + end end t end - def errors - @errors - end - - def params - raise "return uri-value hash" - end - - def validate(index) - raise "validate, return uri" - end - - def build_report(index) - raise "build report, return uri" - end - - def build_compare_report(comparables) - raise "build compare report, return uri" - end - def build() - raise "no comparables" unless @comparables - @validations = Array.new(@comparables.size) - @reports = Array.new(@comparables.size) @errors = {} - to_compare = [] -# LOGGER.info "train-data: "+@train_data.to_s -# LOGGER.info "test-data: "+@test_data.to_s -# LOGGER.info "test-class-data: "+@test_class_data.to_s running = [] - (0..@comparables.size-1).each do |i| - + count = 1 + + @validation_examples.each do |v| + + id = v.title+count.to_s + count += 1 + running << id + LOGGER.debug "Uploading datasets: "+v.title + v.upload_files + v.check_requirements + Thread.new do - running << @comparables[i]+i.to_s - begin - LOGGER.debug "Validate: "+@comparables[i].to_s - @validations[i] = validate(i) - to_compare << @validations[i] if OpenTox::Utils.is_uri?(@validations[i]) - - begin - LOGGER.debug "Building validation-report for "+@validations[i].to_s+" ("+@comparables[i].to_s+")" - @reports[i] = build_report(i) - rescue => ex - LOGGER.error "validation-report error: "+ex.message - @reports[i] = "error" + + LOGGER.debug "Validate: "+v.title + v.validate + if v.validation_error!=nil + @errors["Error validating "+v.title] = v.validation_error + else + LOGGER.debug "Building report: "+v.title + v.report + if v.report_error!=nil + @errors["Error building report for "+v.title] = v.report_error end - - rescue => ex - LOGGER.error "validation error: "+ex.message - key = "Error validating "+@comparables[i].to_s - @validations[i] = key+" (see below)" - @errors[key] = ex.message - ensure - running.delete(@comparables[i]+i.to_s) end + running.delete(id) end end + wait = 0 while running.size>0 LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+" waiting for "+running.inspect if wait%20==0 @@ -226,430 +205,9 @@ class Nightly sleep 1 end - if to_compare.size>1 - LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": build comparison report" - @comparison_report = build_compare_report(to_compare) - else - LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": nothing to compare" - end - end - end - - class AlgorithmValidationBenchmark < ValidationBenchmark - - def comparable_nice_name - return "algorithm" - end - - def build() - raise "no algs" unless @algs - @comparables = @algs - super - end - - def build_compare_report(comparables) - Util.create_alg_comparison_report(comparables) - end - end - - class ModelValidationBenchmark < ValidationBenchmark - - def comparable_nice_name - return "model" - end - - def build() - raise "no models" unless @models - @comparables = @models - super - end - - def build_compare_report(comparables) - "model comparsion report not available yet" #Util.create_model_comparison_report(comparables) - end - end - - class TestModelValidationBenchmark < ModelValidationBenchmark - - def info - [ training_test_info ] - end - - def training_test_info - "This is a test set validation of existing models. "+ - "The model is used to predict the test dataset. Evaluation is done by comparing the model predictions "+ - "to the actual test values (in the test (target) dataset)." - end - - def info_table_title - "Validation params" - end - - def params - p = { "test_dataset_uri" => @test_data } - p["test_target_dataset_uri"] = @test_class_data if @test_class_data - return p - end - - def validate(index) - Util.validate_model(@test_data, @test_class_data, @models[index]) - end - - def build_report(index) - Util.create_report(@validations[index]) - end - - def build() - raise "no test data" unless @test_data - super - end - end - - class TrainingTestValidationBenchmark < AlgorithmValidationBenchmark - - def info - [ training_test_info ] - end - - def training_test_info - "This is a training test set validation. It builds a model with an algorithm and the training dataset. "+ - "The model is used to predict the test dataset. Evaluation is done by comparing the model predictions "+ - "to the actual test values (in the test target dataset)." - end - - def info_table_title - "Validation params" - end - - def params - p = { "training_dataset_uri" => @train_data, "test_dataset_uri" => @test_data, - "prediction_feature" => @pred_feature } - p["test_target_dataset_uri"] = @test_class_data if @test_class_data - return p - end - - def validate(index) - Util.validate_alg(@train_data, @test_data, @test_class_data, - @algs[index], @pred_feature, @alg_params[index]) - end - - def build_report(index) - Util.create_report(@validations[index]) - end - - def build() - raise "no train data" unless @train_data - raise "no test data" unless @test_data - raise "no pred feature" unless @pred_feature - super - end - end - - class CrossValidationBenchmark < AlgorithmValidationBenchmark - - def info - [ training_test_info ] - end - - def training_test_info - "This is a cross-validation." - end - - def info_table_title - "Cross-validation params" + LOGGER.debug self.class.to_s.gsub(/Nightly::/, "")+": build comparison report" + @comparison_report = ValidationExamples::Util.build_compare_report(@validation_examples) end - - def params - p = { "dataset_uri" => @data, "prediction_feature" => @pred_feature, - "num_folds" => @num_folds, "random_seed" => @random_seed, "stratified" => @stratified} - return p - end - - def validate(index) - Util.cross_validate_alg(@data, @algs[index], @pred_feature, - @num_folds, @random_seed, @stratified, @alg_params[index]) - end - - def build_report(index) - Util.create_report(@validations[index], "crossvalidation") - end - - def build() - raise "no data" unless @data - raise "no pred feature" unless @pred_feature - @num_folds = 10 unless @num_folds - @random_seed = 1 unless @random_seed - @stratified = false unless @stratified - super - end - end - - class HamsterCrossvalidationBenchmark < CrossValidationBenchmark - - @@dataset_service = @@config[:services]["opentox-dataset"] - @@file=File.new("data/hamster_carcinogenicity.yaml","r") - @@file_type="text/x-yaml" - @@lazar_server = @@config[:services]["opentox-algorithm"] - - def title() - "Classification, Crossvalidation, Hamster carcinogenicity" - end - - def info - res = [ "A crossvalidation using the hamster carcinogenicity dataset." ] + super - return res - end - - def build() - @algs = [ - #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]), - #File.join(@@lazar_server,"lazar"), - "http://ot-test.in-silico.ch/algorithm/lazar", - #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]), - #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]), - ] - @alg_params = [ - #nil, - #"feature_generation_uri="+File.join(@@lazar_server,"fminer"), - "feature_generation_uri=http://ot-test.in-silico.ch/algorithm/fminer", - nil, - nil - ] - #@pred_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" - @pred_feature = "http://ot-test.in-silico.ch/toxcreate/feature#Hamster%20Carcinogenicity%20(CPDB/DSSTOX)" - @data = "http://ot-test.in-silico.ch/dataset/1" - #LOGGER.debug "upload hamster datasets" - #@data = Util.upload_dataset(@@dataset_service, @@file, @@file_type).chomp("\n") - super - end - end - - class HamsterTrainingTestBenchmark < TrainingTestValidationBenchmark - - @@dataset_service = @@config[:services]["opentox-dataset"] - @@file=File.new("data/hamster_carcinogenicity.yaml","r") - @@file_type="text/x-yaml" - @@lazar_server = @@config[:services]["opentox-algorithm"] - - def title() - "Classification, Algorithm training test validation, Hamster carcinogenicity" - end - - def info - res = [ "A simple binary classification task using the hamster carcinogenicity dataset." ] + super - return res - end - - def build() - @algs = [ - #File.join(@@config[:services]["opentox-majority"],["/class/algorithm"]), - File.join(@@lazar_server,"lazar"), - #"http://188.40.32.88/algorithm/lazar", - ] - @alg_params = [ - #nil, - "feature_generation_uri="+File.join(@@lazar_server,"fminer"), - #"feature_generation_uri=http://188.40.32.88/algorithm/fminer", - ] - - LOGGER.debug "prepare hamster datasets" - - @pred_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" - @test_class_data = Util.upload_dataset(@@dataset_service, @@file, @@file_type).chomp("\n") - #@pred_feature = "http://188.40.32.88/toxcreate/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" - #@test_class_data = "http://188.40.32.88/dataset/57" - - split = Util.split_dataset(@test_class_data, @pred_feature, 0.9, 1) - @train_data = split[0].to_s - @test_data = split[1].to_s - raise "could not split "+@train_data.to_s+" "+@test_data.to_s unless OpenTox::Utils.is_uri?(@train_data) and OpenTox::Utils.is_uri?(@test_data) - super - end - end - - - class MiniRegressionBenchmark < TrainingTestValidationBenchmark - - def title - "Regression, Algorithm training test validation, tiny test dataset" - end - - def info - res = [ "A very small regression task, using the training dataset as test set." ] + super - return res - end - - def build() - @algs = [ - "http://opentox.ntua.gr:3003/algorithm/mlr", - "http://opentox.ntua.gr:3004/algorithm/mlr", - "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNregression", - File.join(@@config[:services]["opentox-majority"],["/regr/algorithm"]) - ] - @alg_params = [ nil, "dataset_service=http://ambit.uni-plovdiv.bg:8080/ambit2/dataset", nil] - @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141" - super - end - end - - class CacoAlgsRegressionBenchmark < TrainingTestValidationBenchmark - - def title - "Regression, Algorithm training test validation, Caco-2 dataset" - end - - def info - res = [ "Training test set validation on caco2 dataset." ] + super - return res - end - - def build() - @algs = [ - "http://opentox.ntua.gr:3003/algorithm/mlr", - "http://opentox.ntua.gr:3004/algorithm/mlr", - "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/LR", - ] - @alg_params = [ nil, nil] - @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798" - @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353" - @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510" - super - end - end - - - class CacoModelsRegressionBenchmark < TestModelValidationBenchmark - - def title - "Regression, Model test set validation, Caco-2 dataset (ambit.uni-plovdiv)" - end - - def info - res = [ "Valdation of two identical(?) mlr models on caco-2 dataset." ] + super - return res - end - - def build() - @models = [ - "http://ambit.uni-plovdiv.bg:8080/ambit2/model/259260", - "http://opentox.ntua.gr:3003/model/195", - ] - #@test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798" - @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353" - super - end - end - - - class CacoModelsRegressionBenchmark2 < TestModelValidationBenchmark - - def title - "Regression, Model test set validation, Caco-2 dataset (apps.ideaconsult)" - end - - def info - res = [ "One more caco2 dataset at another location" ] + super - return res - end - - def build() - @models = [ - "http://apps.ideaconsult.net:8080/ambit2/model/33", - ] - @test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" - #prediction_feature=http://apps.ideaconsult.net:8080/ambit2/feature/22200 - super - end - end - - - - class FatheadRegressionBenchmark < TrainingTestValidationBenchmark - - def title - "Training test set validation, regression with fathead minnow dataset" - end - - def info - res = [ "This is the regression use case used in D2.2. "+ - "The task is to predict LC50 values of the well known Fathead Minnow Acute Toxicity dataset. "+ - "JOELIB was used to compute numerical descriptors as features." ] + super - return res - end - - def build() - @algs = [ - "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNregression", - "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/M5P", - "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/GaussP" - ] - @alg_params = [nil, nil, nil] - @train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/639" - @test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/640" - @pred_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/264185" - super - end - end - - - - - class Util - @@validation_service = @@config[:services]["opentox-validation"] - - def self.upload_dataset(dataset_service, file, file_type) - raise "File not found: "+file.path.to_s unless File.exist?(file.path) - data = File.read(file.path) - data_uri = OpenTox::RestClientWrapper.post dataset_service, {:content_type => file_type}, data - #data_uri = OpenTox::Task.find(data_uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(data_uri) - return data_uri.to_s - end - - def self.split_dataset(data_uri, feature, split_ratio, random_seed) - res = OpenTox::RestClientWrapper.post File.join(@@validation_service,'plain_training_test_split'), { :dataset_uri => data_uri, :prediction_feature=>feature, :split_ratio=>split_ratio, :random_seed=>random_seed} - return res.split("\n") - end - - def self.validate_alg(train_data, test_data, test_class_data, alg, feature, alg_params) - uri = OpenTox::RestClientWrapper.post @@validation_service, { :training_dataset_uri => train_data, :test_dataset_uri => test_data, - :test_target_dataset_uri => test_class_data, - :algorithm_uri => alg, :prediction_feature => feature, :algorithm_params => alg_params } - #LOGGER.info "waiting for validation "+uri.to_s - #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri) - #LOGGER.info "validaiton done "+uri.to_s - return uri.to_s - end - - def self.validate_model(test_data, test_class_data, model) - uri = OpenTox::RestClientWrapper.post @@validation_service, { :test_dataset_uri => test_data, - :test_target_dataset_uri => test_class_data, :model_uri => model } - return uri.to_s - end - - def self.cross_validate_alg(data, alg, feature, folds, seed, stratified, alg_params) - uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"crossvalidation"), { :dataset_uri => data, - :algorithm_uri => alg, :prediction_feature => feature, :algorithm_params => alg_params, :num_folds => folds, - :random_seed => seed, :stratified => stratified } - #LOGGER.info "waiting for validation "+uri.to_s - #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri) - #LOGGER.info "validaiton done "+uri.to_s - return uri.to_s - end - - def self.create_report(validation, type="validation") - uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"report/"+type.to_s), { :validation_uris => validation } - #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri) - return uri.to_s - end - - def self.create_alg_comparison_report(validations) - uri = OpenTox::RestClientWrapper.post File.join(@@validation_service,"report/algorithm_comparison"), { :validation_uris => validations.join("\n") } - #uri = OpenTox::Task.find(uri).wait_for_resource.to_s if OpenTox::Utils.task_uri?(uri) - return uri.to_s - end - end end
\ No newline at end of file diff --git a/nightly/nightly_application.rb b/nightly/nightly_application.rb index c366024..69b4614 100644 --- a/nightly/nightly_application.rb +++ b/nightly/nightly_application.rb @@ -1,8 +1,8 @@ require "nightly/nightly.rb" -get '/build_nightly/?' do +post '/build_nightly/?' do dry_run = params[:dry_run]!=nil and params[:dry_run] - Nightly.build_nightly(dry_run) + Nightly.build_nightly(params["select"],dry_run) end get '/css_style_sheet/?' do diff --git a/report/report_test.rb b/report/report_test.rb index 7a884b1..d580045 100644 --- a/report/report_test.rb +++ b/report/report_test.rb @@ -9,6 +9,7 @@ LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S " class Reports::ApplicationTest < Test::Unit::TestCase include Rack::Test::Methods + include Lib::TestUtil def app Sinatra::Application @@ -37,10 +38,13 @@ class Reports::ApplicationTest < Test::Unit::TestCase #post 'http://ot.validation.de/report/crossvalidation',:validation_uris=>"http://ot.validation.de/crossvalidation/1" #uri = last_response.body.to_s - val_uris = ["http://ot.validation.de/crossvalidation/7","http://ot.validation.de/crossvalidation/8" ] + val_uris = ["http://ot.validation.de/crossvalidation/32"] #,"http://ot.validation.de/crossvalidation/8" ] - post 'http://ot.validation.de/report/algorithm_comparison',:validation_uris=>val_uris.join("\n") - uri = last_response.body.to_s + post 'http://ot.validation.de/report/crossvalidation',:validation_uris=>val_uris.join("\n") + uri = wait_for_task(last_response.body.to_s) + puts uri + id = uri.squeeze("/").split("/")[-1] + get '/report/crossvalidation/'+id,nil,'HTTP_ACCEPT' => "text/html" puts uri #rep = Reports::ReportService.new("http://some.location") diff --git a/test/test_application.rb b/test/test_application.rb new file mode 100644 index 0000000..cbc1057 --- /dev/null +++ b/test/test_application.rb @@ -0,0 +1,17 @@ + +post '/test_validation/?' do + validationExamples = ValidationExamples.select(params[:select]) + return "please \"select\" a single validation example:\n"+ValidationExamples.list if validationExamples.size!=1 or validationExamples[0].size!=1 + OpenTox::Task.as_task do + v = validationExamples[0][0] + ex = v.new + ex.upload_files + ex.check_requirements + ex.validate + raise ex.validation_error if ex.validation_error + ex.report if params[:report] + raise ex.report_error if ex.report_error + ex.validation_uri + (params[:report] ? ","+ex.report_uri : "") + end +end + diff --git a/test/test_examples.rb b/test/test_examples.rb new file mode 100644 index 0000000..d08e358 --- /dev/null +++ b/test/test_examples.rb @@ -0,0 +1,214 @@ + +require 'test/test_examples_util.rb' + +class Class + def humanize + self.to_s.gsub(/.*::/, "").gsub(/([^^A-Z_])([A-Z])/, '\1-\2').gsub(/_/,"-") + end +end + +module ValidationExamples + + ######################################################################################################## + + class ABSTRACTHamsterSplit < SplitTestValidation + def initialize + @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") + @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + end + end + + class LazarHamsterSplit < ABSTRACTHamsterSplit + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + class MajorityHamsterSplit < ABSTRACTHamsterSplit + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + ######################################################################################################## + + class ABSTRACTHamsterTrainingTest < TrainingTestValidation + def initialize + @test_target_dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") + @training_dataset_file = File.new("data/hamster_carcinogenicity.train.yaml","r") + @test_dataset_file = File.new("data/hamster_carcinogenicity.test.yaml","r") + @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + end + end + + class MajorityHamsterTrainingTest < ABSTRACTHamsterTrainingTest + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + class LazarHamsterTrainingTest < ABSTRACTHamsterTrainingTest + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + ######################################################################################################## + + class ABSTRACTHamsterCrossvalidation < CrossValidation + def initialize + @dataset_file = File.new("data/hamster_carcinogenicity.yaml","r") + @prediction_feature = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + end + end + + class MajorityHamsterCrossvalidation < ABSTRACTHamsterCrossvalidation + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + class LazarHamsterCrossvalidation < ABSTRACTHamsterCrossvalidation + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + ######################################################################################################## + + class ABSTRACT_ISTHamsterCrossvalidationInsilico < CrossValidation + def initialize + @dataset_uri = "http://webservices.in-silico.ch/dataset/108" + @prediction_feature = "http://toxcreate.org/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" + end + end + + class MajorityISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") + super + end + end + + class LazarISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + class ISTLazarISTHamsterCrossvalidation < ABSTRACT_ISTHamsterCrossvalidationInsilico + def initialize + @algorithm_uri = "http://webservices.in-silico.ch/algorithm/lazar" + @algorithm_params = "feature_generation_uri=http://webservices.in-silico.ch/algorithm/fminer" + super + end + end + + + ######################################################################################################## + + class LR_AmbitCacoModel < ModelValidation + def initialize + @model_uri = "http://apps.ideaconsult.net:8080/ambit2/model/33" + @test_dataset_uri = "http://apps.ideaconsult.net:8080/ambit2/dataset/R545" + #@prediction_feature=http://apps.ideaconsult.net:8080/ambit2/feature/22200 + end + end + + ######################################################################################################## + + class ABSTRACTCacoTrainingTest < TrainingTestValidation + def initialize + @training_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798" + @test_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R8353" + @prediction_feature = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510" + end + end + + class LR_AmbitCacoTrainingTest < ABSTRACTCacoTrainingTest + def initialize + @algorithm_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/LR" + super + end + end + + class MLR_NTUA_CacoTrainingTest < ABSTRACTCacoTrainingTest + def initialize + @algorithm_uri = "http://opentox.ntua.gr:3003/algorithm/mlr" + super + end + end + + class MLR_NTUA2_CacoTrainingTest < ABSTRACTCacoTrainingTest + def initialize + @algorithm_uri = "http://opentox.ntua.gr:3004/algorithm/mlr" + super + end + end + + class MajorityCacoTrainingTest < ABSTRACTCacoTrainingTest + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + ######################################################################################################## + + @@list = { + "1" => [ LazarHamsterSplit, MajorityHamsterSplit ], + "1a" => [ LazarHamsterSplit ], + "1b" => [ MajorityHamsterSplit ], + + "2" => [ LazarHamsterTrainingTest, MajorityHamsterTrainingTest ], + "2a" => [ LazarHamsterTrainingTest ], + "2b" => [ MajorityHamsterTrainingTest ], + + "3" => [ LazarHamsterCrossvalidation, MajorityHamsterCrossvalidation ], + "3a" => [ LazarHamsterCrossvalidation ], + "3b" => [ MajorityHamsterCrossvalidation ], + + "4" => [ MajorityISTHamsterCrossvalidation, LazarISTHamsterCrossvalidation, ISTLazarISTHamsterCrossvalidation ], + "4a" => [ MajorityISTHamsterCrossvalidation ], + "4b" => [ LazarISTHamsterCrossvalidation ], + "4c" => [ ISTLazarISTHamsterCrossvalidation ], + + "5a" => [ LR_AmbitCacoModel ], + + "6" => [ LR_AmbitCacoTrainingTest, MLR_NTUA_CacoTrainingTest, MLR_NTUA2_CacoTrainingTest, MajorityCacoTrainingTest ], + "6a" => [ LR_AmbitCacoTrainingTest ], + "6b" => [ MLR_NTUA_CacoTrainingTest ], + "6c" => [ MLR_NTUA2_CacoTrainingTest ], + "6d" => [ MajorityCacoTrainingTest ], + } + + def self.list + @@list.sort.collect{|k,v| k+":\t"+v.collect{|vv| vv.humanize}.join("\n\t")+"\n"} #.join("\n") + end + + def self.select(csv_keys) + res = [] + if csv_keys!=nil and csv_keys.size>0 + csv_keys.split(",").each do |k| + raise "no key "+k.to_s unless @@list.has_key?(k) + res << @@list[k] + end + end + return res + end + +end + +#puts ValidationExamples.list +#puts ValidationExamples.select("1,2a").inspect
\ No newline at end of file diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb new file mode 100644 index 0000000..49e0336 --- /dev/null +++ b/test/test_examples_util.rb @@ -0,0 +1,194 @@ + +module ValidationExamples + + class Util + + @@dataset_uris = {} + + def self.upload_dataset(file, dataset_service=@@config[:services]["opentox-dataset"], file_type="text/x-yaml") + raise "File not found: "+file.path.to_s unless File.exist?(file.path) + if @@dataset_uris[file.path.to_s]==nil + data = File.read(file.path) + data_uri = OpenTox::RestClientWrapper.post(dataset_service,{:content_type => file_type},data).to_s.chomp + @@dataset_uris[file.path.to_s] = data_uri + LOGGER.debug "uploaded dataset: "+data_uri + else + LOGGER.debug "file already uploaded: "+@@dataset_uris[file.path.to_s] + end + return @@dataset_uris[file.path.to_s] + end + + def self.build_compare_report(validation_examples) + + @comp = validation_examples[0].algorithm_uri==nil ? :model_uri : :algorithm_uri + return nil if @comp == :model_uri + to_compare = [] + validation_examples.each do |v| + to_compare << v.validation_uri if v.validation_uri and v.validation_error==nil + end + return nil if to_compare.size < 2 + begin + return validation_post "report/algorithm_comparison",{ :validation_uris => to_compare.join("\n") } + rescue => ex + return "error creating comparison report "+ex.message + end + end + + def self.validation_post(uri, params) + if $test_case + #puts "posting: "+uri+","+params.inspect + $test_case.post uri,params + return wait($test_case.last_response.body) + else + return OpenTox::RestClientWrapper.post(File.join(@@config[:services]["opentox-validation"],uri),params) + end + end + + def self.wait(uri) + if OpenTox::Utils.task_uri?(uri) + task = OpenTox::Task.find(uri) + task.wait_for_completion + raise "task failed: "+uri.to_s+", error is:\n"+task.description if task.error? + uri = task.resultURI + end + uri + end + + end + + class ValidationExample + + #params + attr_accessor :name, + :prediction_feature, + :algorithm_uri, + :model_uri, + :test_dataset_uri, + :test_dataset_file, + :test_target_dataset_uri, + :test_target_dataset_file, + :training_dataset_uri, + :training_dataset_file, + :dataset_uri, + :dataset_file, + :algorithm_params, + :split_ratio, + :random_seed, + :num_folds, + :stratified + #results + attr_accessor :validation_uri, + :report_uri, + :validation_error, + :report_error + + def upload_files + [[:test_dataset_uri, :test_dataset_file], + [:test_target_dataset_uri, :test_target_dataset_file], + [:training_dataset_uri, :training_dataset_file], + [:dataset_uri, :dataset_file]].each do |a| + uri = a[0] + file = a[1] + if send(uri)==nil and send(file)!=nil + send("#{uri.to_s}=".to_sym, Util.upload_dataset(send(file))) + end + end + end + + def check_requirements + params.each do |r| + raise "values not set: "+r.to_s if send(r)==nil + end + end + + def report + begin + @report_uri = Util.validation_post '/report/'+report_type,{:validation_uris => @validation_uri} + rescue => ex + @report_error = ex.message + end + end + + def validate + begin + @validation_uri = Util.validation_post '/'+validation_type, get_params + rescue => ex + @validation_error = ex.message + end + end + + def title + self.class.humanize + end + + protected + def report_type + "validation" + end + + def validation_type + "" + end + + def get_params + p = {} + ( params + opt_params ).each do |pp| + p[pp] = send(pp) if send(pp)!=nil + end + return p + end + end + + class ModelValidation < ValidationExample + + def params + [:model_uri, :test_dataset_uri] + end + + def opt_params + [ :prediction_feature, :test_target_dataset_uri ] + end + end + + class TrainingTestValidation < ValidationExample + def params + [:algorithm_uri, :training_dataset_uri, :test_dataset_uri, :prediction_feature] + end + + def opt_params + [ :algorithm_params, :test_target_dataset_uri ] + end + end + + class SplitTestValidation < ValidationExample + def params + [:algorithm_uri, :dataset_uri, :prediction_feature] + end + + def opt_params + [ :algorithm_params, :split_ratio, :random_seed ] + end + + def validation_type + "training_test_split" + end + end + + class CrossValidation < ValidationExample + def params + [:algorithm_uri, :dataset_uri, :prediction_feature] + end + + def opt_params + [ :algorithm_params, :num_folds, :stratified, :random_seed ] + end + + def report_type + "crossvalidation" + end + + def validation_type + "crossvalidation" + end + end +end
\ No newline at end of file diff --git a/validation/validation_test.rb b/validation/validation_test.rb index fbf8d78..0d59c42 100644 --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -6,635 +6,45 @@ require 'application.rb' require 'test/unit' require 'rack/test' require 'lib/test_util.rb' +require 'test/test_examples.rb' LOGGER = Logger.new(STDOUT) LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S " -class Example - attr_accessor :alg, :train_data, :test_data, :model, :pred_data, :act_feat, :pred_feat, :classification, :alg_params, :val, :orig_data, :num_folds, :random_seed -end - class ValidationTest < Test::Unit::TestCase include Rack::Test::Methods include Lib::TestUtil def test_it - #Nightly.build_nightly - #get "/build_nightly" - #get "/nightly" - #get '20',nil,'HTTP_ACCEPT' => "application/rdf+xml" - #puts last_response.body - - #prepare_examples - #do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE - - #ex = ex_maj_class - #ex = ex_maj_regr - #ex = ex_ntua - #ex = ex_ntua2 - #ex = ex_tum - #ex = ex_tum2 - #ex = ex_local - ex = ex_lazar_extern - #ex = ex_ambit - - #create_validation(ex) - #validate_model(ex) - validate_algorithm(ex) - #validate_split(ex) - #xval(ex) - - #test_dataset = OpenTox::Dataset.find ex_ntua2.pred_data - #puts ex_ntua2.pred_data.to_s+", "+test_dataset.compounds.size.to_s+" compounds" - #puts test_dataset.compounds.collect{|c| c.to_s}.join("\n") - end - - def app - Sinatra::Application - end - - def ex_maj_regr - ex = Example.new - ex.classification = true - - ex.alg = File.join(@@config[:services]["opentox-majority"],"regr/algorithm") - #ex.alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") - - ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141" - - return ex - end - - def ex_maj_class - ex = Example.new - ex.classification = true - - ex.alg = File.join(@@config[:services]["opentox-majority"],"class/algorithm") - ex.alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") - - get_hamster(ex) - return ex - end - - def get_hamster(ex) - dataset = @@config[:services]["opentox-dataset"] - ex.orig_data = File.join(dataset,"1") - begin - orig = OpenTox::Dataset.find(ex.orig_data) - raise "not correct, upload" if (orig == nil || orig.compounds.size!=85) - rescue => e - puts e.message - upload_uri = upload_data(dataset, File.new("data/hamster_carcinogenicity.yaml","r")) - ex.orig_data = upload_uri - end - #ex.act_feat = "http://localhost/toxmodel/feature#Hamster Carcinogenicity (DSSTOX/CPDB)" - ex.act_feat = "http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" - - ex.train_data = File.join(dataset,"2") - ex.test_data = File.join(dataset,"3") - begin - train = OpenTox::Dataset.find(ex.train_data) - test = OpenTox::Dataset.find(ex.test_data) - raise "not correct, split "+train.to_s+" "+test.to_s+ - " "+train.compounds.size.to_s+" "+test.compounds.size.to_s if (train == nil || test == nil || train.compounds.size>=85 || test.compounds.size>=train.compounds.size) - rescue => e - puts e.message - post '/plain_training_test_split', { :dataset_uri => ex.orig_data, :prediction_feature=>ex.act_feat, :split_ratio=>0.75, :random_seed=>6} - split = last_response.body.split("\n") - ex.train_data = split[0] - ex.test_data = split[1] - end - end - - def ex_local - ex = Example.new - ex.classification = true - - ex.alg = File.join(@@config[:services]["opentox-algorithm"],"lazar") - ex.alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") - - get_hamster(ex) - - #{:orig => ex.orig_data ,:train => ex.train_data, :test=> ex.test_data}.each do |k,v| - # puts k.to_s+": "+v - # OpenTox::Dataset.find(v).compounds.each do |c| - # puts "XX "+c.to_s if c.to_s =~ /C6H12/ - # end - #end - ex.model = "http://ot.model.de/9" - # example model - #ex.model = "http://opentox.ntua.gr:3000/model/29" - #ex.pred_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/261687" - # example prediction data - #ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/625" - return ex - end - - def ex_lazar_extern - #server = "http://188.40.32.88/" - server = "http://ot-test.in-silico.ch/" - - ex = Example.new - ex.classification = true - ex.alg = server+"algorithm/lazar" - ex.train_data = server+"dataset/1" #51 - ex.test_data = ex.train_data - ex.orig_data = ex.train_data - #ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53 - - ex.alg_params = "feature_generation_uri="+server+"algorithm/fminer" - - ex.act_feat = server+"toxcreate/feature#Hamster%20Carcinogenicity%20(CPDB/DSSTOX)" - - - #ex.act_feat = server+"toxcreate/feature#feature#Hamster Carcinogenicity (DSSTOX/CPDB)" - - #ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53 - #ex.model = "http://apps.ideaconsult.net:8080/ambit2/model/20" - return ex - end - - def ex_ambit - ex = Example.new - ex.classification = false - - #ex.alg = "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/pka" - #ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - #ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141" - - - ex.model = "http://ambit.uni-plovdiv.bg:8080/ambit2/model/259260" - ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/R7798" - #ex. http://ambit.uni-plovdiv.bg:8080/ambit2/feature/255510 - - #ex.alg = "http://ambit.uni-plovdiv.bg:8080/ambit2/algorithm/pka" - #ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - #ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141" - - #ex.alg = "http://apps.ideaconsult.net:8080/ambit2/algorithm/pka" - #ex.train_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/54" #53 - #ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53 - #ex.act_feat = "http://apps.ideaconsult.net:8080/ambit2/feature/22200" #22190" - #ex.model = "http://apps.ideaconsult.net:8080/ambit2/model/20" - - return ex - end - - def ex_ntua2 - ex = Example.new - ex.classification = false - ex.alg = "http://opentox.ntua.gr:3000/algorithm/mlr" - ex.train_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/54" #53 - ex.test_data = "http://apps.ideaconsult.net:8080/ambit2/dataset/55" #53 - ex.act_feat = "http://apps.ideaconsult.net:8080/ambit2/feature/22200" #22190" - - - # example model - #ex.model = "http://opentox.ntua.gr:3000/model/29" - #ex.pred_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/261687" - # example prediction data - #ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/625" - return ex - end - - def ex_ntua - ex = Example.new - ex.classification = false - ex.alg = "http://opentox.ntua.gr:3003/algorithm/mlr" - - #ex.orig_data = "http://apps.ideaconsult.net:8180/ambit2/dataset/52" - - ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - - ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141" - - # example model - ex.model = "http://opentox.ntua.gr:3003/model/201" - #ex.pred_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/264783" - # example prediction data - #ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/656" - # validation - #ex.val = "http://ot.validation.de/21" - return ex - end - - def ex_tum - ex = Example.new - ex.classification = false - ex.alg = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/kNNregression" - #ex.alg = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/M5P" - #ex.alg = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/GaussP" - - #mini - ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141" - - #big - #ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/639" - #ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/640" - #ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/264185" - #ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/264187" #test - - # example model - ex.model = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/model/TUMOpenToxModel_kNN_23" - #ex.model = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/model/TUMOpenToxModel_M5P_13" - #ex.model = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/model/TUMOpenToxModel_GaussP_4" - - #ex.pred_feat = - - # example prediction data - ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/659" #knn - #ex.pred_data = http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/658" #m5p - #ex.pred_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/653" #gaus - - # validataion - # ex.val = "http://ot.validation.de/34" - return ex - end - -def ex_tum2 - ex = Example.new - ex.classification = false - ex.alg = "http://opentox.informatik.tu-muenchen.de:8080/OpenTox-dev/algorithm/J48" - ex.train_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/1027" - ex.test_data = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/1027" - ex.act_feat = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/357709" - return ex - end - -# def test_all_validations -# get '/' -# assert last_response.ok? -# puts "all validations:\n"+last_response.body -# validations = last_response.body.split("\n") -# puts validations.inspect -# if validations && validations.size>0 -# validations.each do |v| -# get v -# puts "get "+v+":\n"+last_response.body -# assert last_response.ok? || last_response.status==202 -# end -# end -# end -# -# def test_all_cvs -# get '/crossvalidations' -# assert last_response.ok? -# puts "all crossvalidations:\n"+last_response.body+"\n" -# crossvalidations = last_response.body.split("\n") -# #puts crossvalidations.inspect -# if crossvalidations && crossvalidations.size>0 -# crossvalidations.each do |v| -# get v -# puts "get "+v+":\n"+last_response.body -# assert last_response.ok? || last_response.status==202 -# end -# end -# end -# - def xval(ex) - begin - #data_uri = upload_data(WS_DATA, FILE) - -# first_validation=nil -# 2.times do - - raise "no orig data" unless ex.orig_data - num_folds = ex.num_folds ? ex.num_folds : 3 - random_seed = ex.random_seed ? ex.random_seed : 1 - - post '/crossvalidation', { :dataset_uri => ex.orig_data, :algorithm_uri => ex.alg, :prediction_feature => ex.act_feat, - :algorithm_params => ex.alg_params, :num_folds => num_folds, :random_seed => random_seed } - - uri = last_response.body - uri = wait_for_task(uri) - puts "crossvalidation: "+uri - - assert last_response.ok? - crossvalidation_id = uri.split("/")[-1] - add_resource("/crossvalidation/"+crossvalidation_id) - puts "id:"+crossvalidation_id - - get '/crossvalidation/'+crossvalidation_id - puts last_response.body - #assert last_response.ok? || last_response.status==202 - - #get '/crossvalidation/'+crossvalidation_id+'/validations' - #puts "validations:\n"+last_response.body - #assert last_response.ok? - # assert last_response.body.split("\n").size == num_folds, "num-folds:"+num_folds.to_s+" but num lines is "+last_response.body.split("\n").size.to_s - -# if first_validation -# # assert that both cross validaitons use the same datasets -# first_validation2 = last_response.body.split("\n")[0].split("/")[-1] -# -# get '/'+first_validation+'/test_dataset_uri' -# assert last_response.ok? -# first_val_test_data = last_response.body -# -# get '/'+first_validation2+'/test_dataset_uri' -# assert last_response.ok? -# first_val2_test_data = last_response.body -# assert first_val_test_data==first_val2_test_data -# end -# first_validation = last_response.body.split("\n")[0].split("/")[-1] -# end - ensure - #delete_resources - end - end -# - def validate_model(ex) - begin -# data_uri_train = upload_data(WS_DATA, DATA_TRAIN, FILE_TRAIN) -# data_uri_test = upload_data(WS_DATA, DATA_TEST, FILE_TEST) -# #data_uri_train = WS_DATA+"/"+DATA_TRAIN -# #data_uri_test = WS_DATA+"/"+DATA_TEST -# -# if WS_FEATURE_ALG -# feature_uri = RestClient.post WS_FEATURE_ALG, :dataset_uri => data_uri_train -# model_uri = RestClient.post(WS_CLASS_ALG,{ :activity_dataset_uri => data_uri_train, :feature_dataset_uri => feature_uri }) -# else -# model_uri = RestClient.post(WS_CLASS_ALG,{ :dataset_uri => data_uri_train }) -# end - -# model_uri = "http://ot.model.de/1" -# data_uri_test = "http://ot.dataset.de/3" - - #model_uri = "http://ot.model.de/7" - #data_uri_test = "http://ot.dataset.de/41" - - # model_uri = "http://opentox.ntua.gr:3000/model/9" - # data_uri_test = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - - raise "model not defined" unless ex.model - - post '', {:test_dataset_uri => ex.test_data, - :test_target_dataset_uri => ex.orig_data, - :model_uri => ex.model} #, :prediction_feature => FEATURE_URI} - - puts last_response.body - #verify_validation - - val_uri = wait_for_task(last_response.body) - puts val_uri - - #get val_uri - #verify_validation(last_response.body) - - ensure - #delete_resources - end - end - - def validate_algorithm(ex) - begin - - #get '/41',nil,'HTTP_ACCEPT' => "application/rdf+xml" #"text/x-yaml" - #puts last_response.body - - #data_uri_train = upload_data(WS_DATA, FILE_TRAIN) - #data_uri_test = upload_data(WS_DATA, FILE_TEST) - - #data_uri_train = WS_DATA+"/"+DATA_TRAIN - #data_uri_test = WS_DATA+"/"+DATA_TEST - - -# data_uri_train="http://ot.dataset.de/57" -# data_uri_test="http://ot.dataset.de/56" -# feature_uri = FEATURE_URI -# algorithm_uri = WS_CLASS_ALG -# algorithm_params="feature_generation_uri="+WS_FEATURE_ALG - - #data_uri_train="http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" - #data_uri_test=data_uri_train - #feature_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141" - #algorithm_uri = "http://opentox.ntua.gr:3000/algorithm/mlr" - #algorithm_params=nil - - #post '', { :training_dataset_uri => data_uri_train, :test_dataset_uri => data_uri_test, - #:algorithm_uri => algorithm_uri, :prediction_feature => feature_uri, :algorithm_params => algorithm_params } - -# uri = OpenTox::RestClientWrapper.post(@@config[:services]["opentox-validation"],{ :training_dataset_uri => ex.train_data, -# :test_dataset_uri => ex.test_data, -# :test_target_dataset_uri => ex.orig_data, -# :algorithm_uri => ex.alg, -# :prediction_feature => ex.act_feat, -# :algorithm_params => ex.alg_params -# },nil,true) - - post '', { :training_dataset_uri => ex.train_data, :test_dataset_uri => ex.test_data, :test_target_dataset_uri => ex.orig_data, - :algorithm_uri => ex.alg, :prediction_feature => ex.act_feat, :algorithm_params => ex.alg_params } - uri = last_response.body.to_s.chomp("\n") - puts uri - uri = wait_for_task(uri) - - puts uri - #get uri - #verify_validation(last_response.body) - #verify_validation - ensure - #delete_resources - end - end - - def create_validation(ex) - -# classification = false -# test_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/342" -# prediction_dataset_uri = "http://ambit.uni-plovdiv.bg:8080/ambit2/dataset/407" -# actual_feature="http://ambit.uni-plovdiv.bg:8080/ambit2/feature/103141" -# predicted_feature = OpenTox::Model::PredictionModel.find("http://opentox.ntua.gr:3000/model/9").predicted_variables -# assert predicted_feature=="http://ambit.uni-plovdiv.bg:8080/ambit2/feature/227289","nope: "+predicted_feature.to_s -# #predicted_feature="http://ambit.uni-plovdiv.bg:8080/ambit2/feature/227289" - -# classification = false -# test_dataset_uri = EX2_TEST -# prediction_dataset_uri = EX2_PRED -# actual_feature= EX2_ACT_FEAT -# predicted_feature = OpenTox::Model::PredictionModel.find(EX2_MODEL).predicted_variables -# assert predicted_feature==EX2_PRED_FEAT,"nope: "+predicted_feature.to_s - -# classification = true -# test_dataset_uri = "http://ot.dataset.de/1" -# prediction_dataset_uri = "http://ot.dataset.de/5" -# actual_feature=URI.encode(FEATURE_URI) -# predicted_feature = OpenTox::Model::PredictionModel.find("http://ot.model.de/1").predicted_variables#_lazar_classification -# assert predicted_feature==URI.encode(FEATURE_URI+"_lazar_classification"), predicted_feature -# #predicted_feature="http://www.epa.gov/NCCT/dsstox/CentralFieldDef.html#ActivityOutcome_CPDBAS_Hamster_lazar_prediction" - - #puts Lib::OTPredictions.new( classification, test_dataset_uri, actual_feature, prediction_dataset_uri, predicted_feature ).compute_stats.each{|key,value| puts key.to_s+" => "+value.to_s } - - - - #predicted_feature = OpenTox::Model::PredictionModel.find(ex.model).predicted_variables - #puts "predicted feature "+predicted_feature.to_s - - assert(predicted_feature==ex.pred_feat,"nope: "+predicted_feature.to_s) if ex.pred_feat +# post "/test_validation",:select=>"6d" #,:report=>"yes,please" +# puts last_response.body - post '/create_validation', { :test_dataset_uri => ex.test_data, :model_uri => ex.model, :prediction_dataset_uri=> ex.pred_data} - puts last_response.body + #run_test("6a") + + #puts Nightly.build_nightly("6") - val_uri = wait_for_task(last_response.body) - puts val_uri - - get val_uri - puts last_response.body - - #puts Lib::OTPredictions.new( ex.classification, ex.test_data, ex.act_feat, ex.pred_data, ex.pred_feat ).compute_stats.each{|key,value| puts key.to_s+" => "+value.to_s } + #prepare_examples + do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE end - - def validate_split(ex) - begin - -# model = OpenTox::Model::PredictionModel.find("http://ot.model.de/66") -# puts model.predicted_variables -# exit - - #data_uri = upload_data(WS_DATA, FILE) - #data_uri = "http://ot.dataset.de/199" #bbrc - #data_uri = "http://ot.dataset.de/67" #hamster - #puts data_uri - - #exit - - #data_uri=WS_DATA+"/"+DATA -# post '/training_test_split', { :dataset_uri => data_uri, :algorithm_uri => WS_CLASS_ALG, :prediction_feature => FEATURE_URI, -# :algorithm_params => "feature_generation_uri="+WS_FEATURE_ALG, :split_ratio=>0.75, :random_seed=>6} - post '/training_test_split', { :dataset_uri => ex.orig_data, :algorithm_uri => ex.alg, :prediction_feature => ex.act_feat, - :algorithm_params => ex.alg_params, :split_ratio=>0.75, :random_seed=>6 } - - puts last_response.body - - val_uri = wait_for_task(last_response.body) - puts val_uri - - get val_uri - puts last_response.body - #verify_validation - ensure - #delete_resources - end + def app + Sinatra::Application end - - def verify_validation(val_yaml) - - puts val_yaml - val = YAML.load(val_yaml) - - puts val.inspect - assert_integer val["num_instances".to_sym],0,1000 - num_instances = val["num_instances".to_sym].to_i - - assert_integer val["num_unpredicted".to_sym],0,num_instances - num_unpredicted = val["num_unpredicted".to_sym].to_i - assert_float val["percent_unpredicted".to_sym],0,100 - assert_float_equal(val["percent_unpredicted".to_sym].to_f,100*num_unpredicted/num_instances.to_f,"percent_unpredicted") - - assert_integer val["num_without_class".to_sym],0,num_instances - num_without_class = val["num_without_class".to_sym].to_i - assert_float val["percent_without_class".to_sym],0,100 - assert_float_equal(val["percent_without_class".to_sym].to_f,100*num_without_class/num_instances.to_f,"percent_without_class") - - class_stats = val["classification_statistics".to_sym] - if class_stats - class_value_stats = class_stats["class_value_statistics".to_sym] - class_values = [] - class_value_stats.each do |cvs| - class_values << cvs["class_value".to_sym] + def run_test(select) + $test_case = self + validationExamples = ValidationExamples.select(select) + validationExamples.each do |vv| + vv.each do |v| + ex = v.new + ex.upload_files + ex.check_requirements + ex.validate + ex.report end - puts class_values.inspect - - confusion_matrix = class_stats["confusion_matrix".to_sym] - confusion_matrix_cells = confusion_matrix["confusion_matrix_cell".to_sym] - predictions = 0 - confusion_matrix_cells.each do |confusion_matrix_cell| - predictions += confusion_matrix_cell["confusion_matrix_value".to_sym].to_i - end - assert_int_equal(predictions, num_instances-num_unpredicted) - else - regr_stats = val["regression_statistics".to_sym] - assert regr_stats!=nil end end - def assert_int_equal(val1,val2,msg_suffix=nil) - assert(val1==val2,msg_suffix.to_s+" not equal: "+val1.to_s+" != "+val2.to_s) - end - - def assert_float_equal(val1,val2,msg_suffix=nil,epsilon=0.0001) - assert((val1-val2).abs<epsilon,msg_suffix.to_s+" not equal: "+val1.to_s+" != "+val2.to_s+", diff:"+(val1-val2).abs.to_s) - end - - def assert_integer(string_val, min=nil, max=nil) - assert string_val.to_i.to_s==string_val.to_s, string_val.to_s+" not an integer" - assert string_val.to_i>=min if min!=nil - assert string_val.to_i<=max if max!=nil - end - - def assert_float(string_val, min=nil, max=nil) - assert( string_val.to_f.to_s==string_val.to_s || (string_val.to_f.to_s==(string_val.to_s+".0")), - string_val.to_s+" not a float (!="+string_val.to_f.to_s+")") - assert string_val.to_f>=min if min!=nil - assert string_val.to_f<=max if max!=nil - end - -# def test_nothing -# -# #puts "testing nothing" -# -# #get '/' -# -# #get '/crossvalidation/loo' -# #get '/training_test_split' -# -# #get '/1',nil,'HTTP_ACCEPT' => "application/rdf+xml" -# #get '/1',nil,'HTTP_ACCEPT' => "text/x-yaml" -# -# -# #get '/crossvalidation/1',nil,'HTTP_ACCEPT' => "application/rdf+xml" -# #get '/crossvalidation/1/statistics',nil,'HTTP_ACCEPT' => "text/x-yaml" -# -# #puts last_response.body -# -# #get '/2' -# #verify_validation(last_response.body) -# -# end - -# private -# def verify_validation (delete=true) -# -# puts "validation: "+last_response.body -# assert last_response.ok? -# validation_id = last_response.body.split("/")[-1] -# -# puts "uri: "+last_response.body -# puts "id:"+validation_id -# add_resource("/"+validation_id) if delete -# -# #get '/'+validation_id,nil,'HTTP_ACCEPT' => "application/rdf+xml" -# get '/'+validation_id,nil,'HTTP_ACCEPT' => "text/x-yaml" -# puts last_response.body -# assert last_response.ok? || last_response.status==202 -# -## ["test_dataset_uri", "model_uri", "prediction_dataset_uri"].each do |t| -## get '/'+validation_id+'/'+t -## puts ""+t+": "+last_response.body -## assert last_response.ok? -## -## content = ext("curl "+last_response.body) -## content = content.split("\n")[0,10].join("\n")+"\n...\n" if content.count("\n")>10 -## puts content -## end -# end - def prepare_examples get '/prepare_examples' end @@ -642,4 +52,5 @@ def ex_tum2 def do_test_examples # USES CURL, DO NOT FORGET TO RESTART get '/test_examples' end + end |