diff options
author | Christoph Helma <helma@in-silico.ch> | 2010-07-20 11:31:27 +0200 |
---|---|---|
committer | Christoph Helma <helma@in-silico.ch> | 2010-07-20 11:31:27 +0200 |
commit | 5f243f1e9e0a8e12cd8a2267bcec3140d21bf445 (patch) | |
tree | e755d51a453ed710dd4d78437b197c408d5687eb | |
parent | 2963e4f7c355394130297ad181cd30f5e78d1017 (diff) | |
parent | 2a4fe800d1fad7da3a835f664ce1af864e827c0f (diff) |
Merge commit 'mguetlein/test'
Conflicts:
application.rb
-rw-r--r-- | EXAMPLES | 19 | ||||
-rw-r--r-- | application.rb | 2 | ||||
-rw-r--r-- | data/IRIS_unitrisk.yaml | 231 | ||||
-rw-r--r-- | example.rb | 5 | ||||
-rw-r--r-- | lib/merge.rb | 10 | ||||
-rw-r--r-- | lib/predictions.rb | 30 | ||||
-rw-r--r-- | lib/validation_db.rb | 3 | ||||
-rw-r--r-- | report/environment.rb | 4 | ||||
-rw-r--r-- | report/report_application.rb | 35 | ||||
-rw-r--r-- | report/report_factory.rb | 8 | ||||
-rw-r--r-- | report/report_persistance.rb | 14 | ||||
-rw-r--r-- | report/report_test.rb | 12 | ||||
-rw-r--r-- | report/validation_data.rb | 8 | ||||
-rw-r--r-- | test/test_examples.rb | 97 | ||||
-rw-r--r-- | test/test_examples_util.rb | 8 | ||||
-rw-r--r-- | validation/validation_application.rb | 24 | ||||
-rw-r--r-- | validation/validation_format.rb | 14 | ||||
-rw-r--r-- | validation/validation_service.rb | 62 | ||||
-rw-r--r-- | validation/validation_test.rb | 26 |
19 files changed, 527 insertions, 85 deletions
@@ -90,6 +90,25 @@ result example (accept-header: text/uri-list) <<< <validation_service>/id_i +create a validation by comparing feature values +----------------------------------------------------- + +>>> curl -X POST -d test_dataset_uri="<test_dataset_uri>" \ + -d test_target_dataset_uri="<dataset_uri>" \ + -d prediction_dataset_uri="<prediction_dataset_uri>" \ + -d prediction_feature="<prediction_feature>" \ + -d predicted_feature="<predicted_feature>" \ + -d classification="true" \ + <validation_service>/validate_datasets + +optional params: +test_target_dataset_uri, default is test_dataset_uri +regression or classification has to be specified + +result example (accept-header: text/uri-list) +<<< <validation_service>/id_i + + validate an algorithm on a dataset via training-test-split ------------------------------------------------------------------- diff --git a/application.rb b/application.rb index f7208a4..a8ae5a8 100644 --- a/application.rb +++ b/application.rb @@ -1,6 +1,6 @@ require 'rubygems' gem 'opentox-ruby-api-wrapper', '= 1.6.0' -[ 'sinatra', 'sinatra/url_for', 'opentox-ruby-api-wrapper', 'logger' ].each do |lib| +[ 'sinatra', 'sinatra/url_for', 'opentox-ruby-api-wrapper' ].each do |lib| require lib end diff --git a/data/IRIS_unitrisk.yaml b/data/IRIS_unitrisk.yaml new file mode 100644 index 0000000..8a6d7c5 --- /dev/null +++ b/data/IRIS_unitrisk.yaml @@ -0,0 +1,231 @@ +--- !ruby/object:OpenTox::Dataset +compounds: +- http://ot-dev.in-silico.ch/compound/InChI=1S/C4H10NO3PS/c1-4(6)5-9(7,8-2)10-3/h1-3H3,(H,5,6,7) +- http://ot-dev.in-silico.ch/compound/InChI=1S/C3H5NO/c1-2-3(4)5/h2H,1H2,(H2,4,5) +- http://ot-dev.in-silico.ch/compound/InChI=1S/C3H3N/c1-2-3-4/h2H,1H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C12H8Cl6/c13-8-9(14)11(16)7-5-2-1-4(3-5)6(7)10(8,15)12(11,17)18/h1-2,4-7H,3H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C6H7N/c7-6-4-2-1-3-5-6/h1-5H,7H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C15H23ClO4S/c1-12(20-21(17)19-10-9-16)11-18-14-7-5-13(6-8-14)15(2,3)4/h5-8,12H,9-11H2,1-4H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/As +- http://ot-dev.in-silico.ch/compound/InChI=1S/C12H10N2/c1-3-7-11(8-4-1)13-14-12-9-5-2-6-10-12/h1-10H/b14-13- +- http://ot-dev.in-silico.ch/compound/InChI=1S/C6H6/c1-2-4-6-5-3-1/h1-6H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C12H12N2/c13-11-5-1-9(2-6-11)10-3-7-12(14)8-4-10/h1-8H,13-14H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C20H12/c1-2-7-17-15(4-1)12-16-9-8-13-5-3-6-14-10-11-18(17)20(16)19(13)14/h1-12H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C7H5Cl3/c8-7(9,10)6-4-2-1-3-5-6/h1-5H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C7H7Cl/c8-6-7-4-2-1-3-5-7/h1-5H,6H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C4H8Cl2O/c5-1-3-7-4-2-6/h1-4H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2H4Cl2O/c3-1-5-2-4/h1-2H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/BrHO3/c2-1(3)4/h(H,2,3,4)/p-1 +- http://ot-dev.in-silico.ch/compound/InChI=1S/CHBrCl2/c2-1(3)4/h1H +- http://ot-dev.in-silico.ch/compound/InChI=1S/CHBr3/c2-1(3)4/h1H +- http://ot-dev.in-silico.ch/compound/InChI=1S/CCl4/c2-1(3,4)5 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C10H6Cl8/c11-3-1-2-4(5(3)12)9(16)7(14)6(13)8(2,15)10(9,17)18/h2-5H,1H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C22H42O4/c1-5-9-13-19(7-3)17-25-21(23)15-11-12-16-22(24)26-18-20(8-4)14-10-6-2/h19-20H,5-18H2,1-4H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C24H38O4/c1-5-9-13-19(7-3)17-27-23(25)21-15-11-12-16-22(21)24(26)28-18-20(8-4)14-10-6-2/h11-12,15-16,19-20H,5-10,13-14,17-18H2,1-4H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/CHBr2Cl/c2-1(3)4/h1H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2H4Br2/c3-1-2-4/h1-2H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2H2Cl2O2/c3-1(4)2(5)6/h1H,(H,5,6) +- http://ot-dev.in-silico.ch/compound/InChI=1S/C12H10Cl2N2/c13-9-5-7(1-3-11(9)15)8-2-4-12(16)10(14)6-8/h1-6H,15-16H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C14H10Cl4/c15-11-5-1-9(2-6-11)13(14(17)18)10-3-7-12(16)8-4-10/h1-8,13-14H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C14H8Cl4/c15-11-5-1-9(2-6-11)13(14(17)18)10-3-7-12(16)8-4-10/h1-8H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C14H9Cl5/c15-11-5-1-9(2-6-11)13(14(17,18)19)10-3-7-12(16)8-4-10/h1-8,13H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2H4Cl2/c3-1-2-4/h1-2H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/CH2Cl2/c2-1-3/h1H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C3H4Cl2/c4-2-1-3-5/h1-2H,3H2/b2-1+ +- http://ot-dev.in-silico.ch/compound/InChI=1S/C4H7Cl2O4P/c1-8-11(7,9-2)10-3-4(5)6/h3H,1-2H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C12H8Cl6O/c13-8-9(14)11(16)5-3-1-2(6-7(3)19-6)4(5)10(8,15)12(11,17)18/h2-7H,1H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C7H6N2O4/c1-5-6(8(10)11)3-2-4-7(5)9(12)13/h2-4H,1H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C4H8O2/c1-2-6-4-3-5-1/h1-4H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C12H12N2/c1-3-7-11(8-4-1)13-14-12-9-5-2-6-10-12/h1-10,13-14H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C3H5ClO/c4-1-3-2-5-3/h3H,1-2H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C9H4Cl3NO2S/c10-9(11,12)16-13-7(14)5-3-1-2-4-6(5)8(13)15/h1-4H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C15H10ClF3N2O6S/c1-28(25,26)20-14(22)10-7-9(3-4-12(10)21(23)24)27-13-5-2-8(6-11(13)16)15(17,18)19/h2-7H,1H3,(H,20,22) +- http://ot-dev.in-silico.ch/compound/InChI=1S/C14H21NO3/c1-10-9-13(11(2)18-10)14(16)15(17-3)12-7-5-4-6-8-12/h9,12H,4-8H2,1-3H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C10H5Cl7/c11-4-2-1-3-5(4)9(15)7(13)6(12)8(3,14)10(9,16)17/h1-5H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C10H5Cl7O/c11-3-1-2(4-5(3)18-4)9(15)7(13)6(12)8(1,14)10(9,16)17/h1-5H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C4Cl6/c5-1(3(7)8)2(6)4(9)10 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C6H6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9/h1-6H/t1-,2-,3-,4+,5+,6?/m0/s1 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C6H6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9/h1-6H/t1-,2-,3+,4+,5-,6? +- http://ot-dev.in-silico.ch/compound/InChI=1S/C6H6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9/h1-6H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C12H2Cl6O2/c13-3-1-5-11(9(17)7(3)15)20-12-6(19-5)2-4(14)8(16)10(12)18/h1-2H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2Cl6/c3-1(4,5)2(6,7)8 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C3H6N6O6/c10-7(11)4-1-5(8(12)13)3-6(2-4)9(14)15/h1-3H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/H4N2/c1-2/h1-2H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C9H14O/c1-7-4-8(10)6-9(2,3)5-7/h4H,5-6H2,1-3H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C17H22N2/c1-18(2)16-9-5-14(6-10-16)13-15-7-11-17(12-8-15)19(3)4/h5-12H,13H2,1-4H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C3H8N2O/c1-3-5(2)4-6/h3H2,1-2H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C8H18N2O/c1-3-5-7-10(9-11)8-6-4-2/h3-8H2,1-2H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C6H14N2O/c1-3-5-8(7-9)6-4-2/h3-6H2,1-2H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C4H10N2O3/c7-3-1-6(5-9)2-4-8/h7-8H,1-4H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C4H10N2O/c1-3-6(4-2)5-7/h3-4H2,1-2H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2H6N2O/c1-4(2)3-5/h1-2H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C12H10N2O/c15-13-14(11-7-3-1-4-8-11)12-9-5-2-6-10-12/h1-10H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C4H8N2O/c7-5-6-3-1-2-4-6/h1-4H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C6HCl5O/c7-1-2(8)4(10)6(12)5(11)3(1)9/h12H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C12H4Cl6/c13-7-1-5(2-8(14)11(7)17)6-3-9(15)12(18)10(16)4-6/h1-4H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C15H16Cl3N3O2/c1-2-4-20(15(22)21-5-3-19-10-21)6-7-23-14-12(17)8-11(16)9-13(14)18/h3,5,8-10H,2,4,6-7H2,1H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C3H6O/c1-3-2-4-3/h3H,2H2,1H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C9H7N/c1-2-6-9-8(4-1)5-3-7-10-9/h1-7H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2H2Cl4/c3-1(4)2(5)6/h1-2H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2H2Cl4/c3-1-2(4,5)6/h1H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C10H8Cl8/c1-4-7(2-11,3-12)9(16)6(14)5(13)8(4,15)10(9,17)18/h5-6H,1-3H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2H3Cl3/c3-1-2(4)5/h2H,1H2 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C6H3Cl3O/c7-3-1-4(8)6(10)5(9)2-3/h1-2,10H +- http://ot-dev.in-silico.ch/compound/InChI=1S/C13H16F3N3O4/c1-3-5-17(6-4-2)12-10(18(20)21)7-9(13(14,15)16)8-11(12)19(22)23/h7-8H,3-6H2,1-2H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C7H5N3O6/c1-4-6(9(13)14)2-5(8(11)12)3-7(4)10(15)16/h2-3H,1H3 +- http://ot-dev.in-silico.ch/compound/InChI=1S/C2H3Cl/c1-2-3/h2H,1H2 +data: + http://ot-dev.in-silico.ch/compound/InChI=1S/C12H8Cl6/c13-8-9(14)11(16)7-5-2-1-4(3-5)6(7)10(8,15)12(11,17)18/h1-2,4-7H,3H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.34e-06 + http://ot-dev.in-silico.ch/compound/InChI=1S/C3H8N2O/c1-3-5(2)4-6/h3H2,1-2H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 7.15e-06 + http://ot-dev.in-silico.ch/compound/InChI=1S/C3H6O/c1-3-2-4-3/h3H,2H2,1H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.17e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2Cl6/c3-1(4,5)2(6,7)8: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.69e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C10H5Cl7O/c11-3-1-2(4-5(3)18-4)9(15)7(13)6(12)8(1,14)10(9,16)17/h1-5H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 6.68e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C15H16Cl3N3O2/c1-2-4-20(15(22)21-5-3-19-10-21)6-7-23-14-12(17)8-11(16)9-13(14)18/h3,5,8-10H,2,4,6-7H2,1H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.14e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C9H7N/c1-2-6-9-8(4-1)5-3-7-10-9/h1-7H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 6.97e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C12H10N2O/c15-13-14(11-7-3-1-4-8-11)12-9-5-2-6-10-12/h1-10H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 7.06e-10 + http://ot-dev.in-silico.ch/compound/InChI=1S/C12H8Cl6O/c13-8-9(14)11(16)5-3-1-2(6-7(3)19-6)4(5)10(8,15)12(11,17)18/h2-7H,1H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.21e-06 + http://ot-dev.in-silico.ch/compound/InChI=1S/CCl4/c2-1(3,4)5: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.41e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C15H23ClO4S/c1-12(20-21(17)19-10-9-16)11-18-14-7-5-13(6-8-14)15(2,3)4/h5-8,12H,9-11H2,1-4H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.12e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/CH2Cl2/c2-1-3/h1H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.47e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C3H5ClO/c4-1-3-2-5-3/h3H,1-2H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.03e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C24H38O4/c1-5-9-13-19(7-3)17-27-23(25)21-15-11-12-16-22(21)24(26)28-18-20(8-4)14-10-6-2/h11-12,15-16,19-20H,5-10,13-14,17-18H2,1-4H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.02e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C7H6N2O4/c1-5-6(8(10)11)3-2-4-7(5)9(12)13/h2-4H,1H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.04e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/CHBr2Cl/c2-1(3)4/h1H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.15e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C6H14N2O/c1-3-5-8(7-9)6-4-2/h3-6H2,1-2H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.54e-06 + http://ot-dev.in-silico.ch/compound/InChI=1S/CHBrCl2/c2-1(3)4/h1H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.1e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C12H2Cl6O2/c13-3-1-5-11(9(17)7(3)15)20-12-6(19-5)2-4(14)8(16)10(12)18/h1-2H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 0.000461 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2H3Cl/c1-2-3/h2H,1H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.36e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C14H8Cl4/c15-11-5-1-9(2-6-11)13(14(17)18)10-3-7-12(16)8-4-10/h1-8H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.05e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/As: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 6.67e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2H4Cl2/c3-1-2-4/h1-2H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.63e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C3H3N/c1-2-3-4/h2H,1H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.83e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C3H6N6O6/c10-7(11)4-1-5(8(12)13)3-6(2-4)9(14)15/h1-3H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.4e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C6H3Cl3O/c7-3-1-4(8)6(10)5(9)2-3/h1-2,10H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.57e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C9H14O/c1-7-4-8(10)6-9(2,3)5-7/h4H,5-6H2,1-3H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.95e-10 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2H2Cl4/c3-1(4)2(5)6/h1-2H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.46e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C12H10N2/c1-3-7-11(8-4-1)13-14-12-9-5-2-6-10-12/h1-10H/b14-13-: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.7e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C4H8O2/c1-2-6-4-3-5-1/h1-4H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.52e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C3H4Cl2/c4-2-1-3-5/h1-2H,3H2/b2-1+: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.7e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.62e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C4H10N2O3/c7-3-1-6(5-9)2-4-8/h7-8H,1-4H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 5.96e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C7H5N3O6/c1-4-6(9(13)14)2-5(8(11)12)3-7(4)10(15)16/h2-3H,1H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.96e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C17H22N2/c1-18(2)16-9-5-14(6-10-16)13-15-7-11-17(12-8-15)19(3)4/h5-12H,13H2,1-4H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 5.11e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C20H12/c1-2-7-17-15(4-1)12-16-9-8-13-5-3-6-14-10-11-18(17)20(16)19(13)14/h1-12H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 8.32e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2H4Br2/c3-1-2-4/h1-2H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.19e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C6H6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9/h1-6H/t1-,2-,3+,4+,5-,6?: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.82e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C10H5Cl7/c11-4-2-1-3-5(4)9(15)7(13)6(12)8(3,14)10(9,16)17/h1-5H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.48e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C6H6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9/h1-6H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.75e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C15H10ClF3N2O6S/c1-28(25,26)20-14(22)10-7-9(3-4-12(10)21(23)24)27-13-5-2-8(6-11(13)16)15(17,18)19/h2-7H,1H3,(H,20,22): + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.23e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C3H5NO/c1-2-3(4)5/h2H,1H2,(H2,4,5): + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.83e-06 + http://ot-dev.in-silico.ch/compound/InChI=1S/C22H42O4/c1-5-9-13-19(7-3)17-25-21(23)15-11-12-16-22(24)26-18-20(8-4)14-10-6-2/h19-20H,5-18H2,1-4H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 9.18e-11 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2H2Cl4/c3-1-2(4,5)6/h1H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 4.41e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/BrHO3/c2-1(3)4/h(H,2,3,4)/p-1: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.56e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2H2Cl2O2/c3-1(4)2(5)6/h1H,(H,5,6): + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.09e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C6H6/c1-2-4-6-5-3-1/h1-6H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 5.63e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C14H10Cl4/c15-11-5-1-9(2-6-11)13(14(17)18)10-3-7-12(16)8-4-10/h1-8,13-14H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.16e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C14H9Cl5/c15-11-5-1-9(2-6-11)13(14(17,18)19)10-3-7-12(16)8-4-10/h1-8,13H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.74e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C10H6Cl8/c11-3-1-2-4(5(3)12)9(16)7(14)6(13)8(2,15)10(9,17)18/h2-5H,1H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.44e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C12H10Cl2N2/c13-9-5-7(1-3-11(9)15)8-2-4-12(16)10(14)6-8/h1-6H,15-16H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 5.14e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C4Cl6/c5-1(3(7)8)2(6)4(9)10: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 8.44e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C12H4Cl6/c13-7-1-5(2-8(14)11(7)17)6-3-9(15)12(18)10(16)4-6/h1-4H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.77e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C4H8Cl2O/c5-1-3-7-4-2-6/h1-4H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.31e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/CHBr3/c2-1(3)4/h1H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 9.1e-10 + http://ot-dev.in-silico.ch/compound/InChI=1S/C13H16F3N3O4/c1-3-5-17(6-4-2)12-10(18(20)21)7-9(13(14,15)16)8-11(12)19(22)23/h7-8H,3-6H2,1-2H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 6.56e-10 + http://ot-dev.in-silico.ch/compound/InChI=1S/C8H18N2O/c1-3-5-7-10(9-11)8-6-4-2/h3-8H2,1-2H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.01e-06 + http://ot-dev.in-silico.ch/compound/InChI=1S/C4H10NO3PS/c1-4(6)5-9(7,8-2)10-3/h1-3H3,(H,5,6,7): + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.36e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2H4Cl2O/c3-1-5-2-4/h1-2H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 5.39e-05 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2H6N2O/c1-4(2)3-5/h1-2H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.89e-05 + http://ot-dev.in-silico.ch/compound/InChI=1S/H4N2/c1-2/h1-2H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 2.65e-06 + http://ot-dev.in-silico.ch/compound/InChI=1S/C2H3Cl3/c3-1-2(4)5/h2H,1H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.2e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C4H8N2O/c7-5-6-3-1-2-4-6/h1-4H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 6.09e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C12H12N2/c13-11-5-1-9(2-6-11)10-3-7-12(14)8-4-10/h1-8H,13-14H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.64e-05 + http://ot-dev.in-silico.ch/compound/InChI=1S/C6H7N/c7-6-4-2-1-3-5-6/h1-5H,7H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.72e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C6H6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9/h1-6H/t1-,2-,3-,4+,5+,6?/m0/s1: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 6.19e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C4H7Cl2O4P/c1-8-11(7,9-2)10-3-4(5)6/h3H,1-2H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.76e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C9H4Cl3NO2S/c10-9(11,12)16-13-7(14)5-3-1-2-4-6(5)8(13)15/h1-4H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.37e-10 + http://ot-dev.in-silico.ch/compound/InChI=1S/C10H8Cl8/c1-4-7(2-11,3-12)9(16)6(14)5(13)8(4,15)10(9,17)18/h5-6H,1-3H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 7.77e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C12H12N2/c1-3-7-11(8-4-1)13-14-12-9-5-2-6-10-12/h1-10,13-14H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.19e-07 + http://ot-dev.in-silico.ch/compound/InChI=1S/C7H7Cl/c8-6-7-4-2-1-3-5-7/h1-5H,6H2: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.87e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C6HCl5O/c7-1-2(8)4(10)6(12)5(11)3(1)9/h12H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.13e-08 + http://ot-dev.in-silico.ch/compound/InChI=1S/C14H21NO3/c1-10-9-13(11(2)18-10)14(16)15(17-3)12-7-5-4-6-8-12/h9,12H,4-8H2,1-3H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 3.42e-09 + http://ot-dev.in-silico.ch/compound/InChI=1S/C4H10N2O/c1-3-6(4-2)5-7/h3-4H2,1-2H3: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 4.21e-05 + http://ot-dev.in-silico.ch/compound/InChI=1S/C7H5Cl3/c8-7(9,10)6-4-2-1-3-5-6/h1-5H: + - http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk: 1.84e-06 +features: +- http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk +title: IRIS unit risk @@ -7,11 +7,13 @@ class Example @@file_type="text/x-yaml" @@model=File.join @@config[:services]["opentox-model"],"1" @@feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)") + @@predicted_feature= URI.encode("http://localhost/toxmodel/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)_lazar_classification") @@alg = File.join @@config[:services]["opentox-algorithm"],"lazar" @@alg_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") @@data=File.join @@config[:services]["opentox-dataset"],"1" @@train_data=File.join @@config[:services]["opentox-dataset"],"2" @@test_data=File.join @@config[:services]["opentox-dataset"],"3" + @@prediction_data=File.join @@config[:services]["opentox-dataset"],"5" @@css_file="http://apps.ideaconsult.net:8080/ToxPredict/style/global.css" @@summary="" @@ -39,7 +41,8 @@ class Example "validation_report_id" => "1", "crossvalidation_report_id" => "1", "css_file" => @@css_file, - } + "prediction_dataset_uri" => @@prediction_data, + "predicted_feature" => @@predicted_feature } sub.each do |k,v| res.gsub!(/<#{k}>/,v) diff --git a/lib/merge.rb b/lib/merge.rb index 7e05cd8..527415e 100644 --- a/lib/merge.rb +++ b/lib/merge.rb @@ -119,7 +119,9 @@ module Lib elsif value1.is_a?(Array) raise "non-numerical arrays not yet supported" else - if value1.to_s != value2.to_s + if value1==nil && value2==nil + value = nil + elsif value1.to_s != value2.to_s value = value1.to_s + "/" + value2.to_s else value = value2.to_s @@ -174,14 +176,14 @@ module Lib class MergeTest - attr_accessor :string, :integer, :float, :hash_value, :float, :float_array, :float_variance, :float_array_variance + attr_accessor :string, :integer, :float, :hash_value, :float, :float_array, :float_variance, :float_array_variance, :is_nil AVG = [:float, :float_array, :int_key ] SUM = [:integer ] - ELSE = [:string, :hash_value] + ELSE = [:string, :hash_value, :is_nil] def to_s - res = [:string, :integer, :float, :hash_value, :float_array].collect do |var| + res = [:is_nil, :string, :integer, :float, :hash_value, :float_array].collect do |var| variance = nil begin variance = "+-"+send((var.to_s+"_variance")).inspect if AVG.index(var)!=nil diff --git a/lib/predictions.rb b/lib/predictions.rb index a183534..f6351f8 100644 --- a/lib/predictions.rb +++ b/lib/predictions.rb @@ -97,6 +97,12 @@ module Lib @variance_predicted = 0 @variance_actual = 0 + + @sum_actual = 0 + @sum_predicted = 0 + @sum_multiply = 0 + @sum_squares_actual = 0 + @sum_squares_predicted = 0 end end @@ -134,6 +140,12 @@ module Lib @prediction_mean, old_prediction_mean, predicted_value ) @variance_actual = Util.compute_variance( @variance_actual, @num_predicted, @actual_mean, old_actual_mean, actual_value ) + + @sum_actual += actual_value + @sum_predicted += predicted_value + @sum_multiply += (actual_value*predicted_value) + @sum_squares_actual += actual_value**2 + @sum_squares_predicted += predicted_value**2 end end end @@ -418,9 +430,23 @@ module Lib Math.sqrt(@sum_abs_error / (@num_with_actual_value - @num_unpredicted).to_f) end + def sum_squared_error + return @sum_squared_error + end + def r_square - return 0 if @variance_actual==0 - return @variance_predicted / @variance_actual + return sample_correlation_coefficient ** 2 + end + + def sample_correlation_coefficient + # formula see http://en.wikipedia.org/wiki/Correlation_and_dependence#Pearson.27s_product-moment_coefficient + return ( @num_predicted * @sum_multiply - @sum_actual * @sum_predicted ) / + ( Math.sqrt( @num_predicted * @sum_squares_actual - @sum_actual**2 ) * + Math.sqrt( @num_predicted * @sum_squares_predicted - @sum_predicted**2 ) ) + end + + def total_sum_of_squares + return @variance_actual * ( @num_predicted - 1 ) end def target_variance_predicted diff --git a/lib/validation_db.rb b/lib/validation_db.rb index aea036f..1152abf 100644 --- a/lib/validation_db.rb +++ b/lib/validation_db.rb @@ -46,7 +46,8 @@ module Lib VAL_CLASS_PROPS_EXTENDED = VAL_CLASS_PROPS + [:accuracy] # :regression_statistics - VAL_REGR_PROPS = [ :root_mean_squared_error, :mean_absolute_error, :r_square, :target_variance_actual, :target_variance_predicted ] + VAL_REGR_PROPS = [ :root_mean_squared_error, :mean_absolute_error, :r_square, + :target_variance_actual, :target_variance_predicted, :sum_squared_error, :sample_correlation_coefficient ] CROSS_VAL_PROPS = [:dataset_uri, :num_folds, :stratified, :random_seed] CROSS_VAL_PROPS_REDUNDANT = [:crossvalidation_uri, :algorithm_uri, :created_at] + CROSS_VAL_PROPS diff --git a/report/environment.rb b/report/environment.rb index 7c9aa8b..714cebe 100644 --- a/report/environment.rb +++ b/report/environment.rb @@ -1,7 +1,7 @@ ['rubygems', 'logger', 'fileutils', 'sinatra', 'sinatra/url_for', 'rest_client', - 'yaml', 'opentox-ruby-api-wrapper', 'fileutils', 'mime/types', 'abbrev', - 'rexml/document', 'active_record', 'ar-extensions', 'ruby-plot'].each do |g| + 'yaml', 'fileutils', 'mime/types', 'abbrev', + 'rexml/document', 'ruby-plot', 'active_record', 'ar-extensions', 'opentox-ruby-api-wrapper' ].each do |g| require g end gem 'ruby-plot', '= 0.0.2' diff --git a/report/report_application.rb b/report/report_application.rb index 27c44b6..efa0298 100644 --- a/report/report_application.rb +++ b/report/report_application.rb @@ -60,14 +60,21 @@ get '/report/:type/:id' do end #request.env['HTTP_ACCEPT'] = "application/pdf" - report = rs.get_report(params[:type],params[:id],accept_header) - format = Reports::ReportFormat.get_format(accept_header) - content_type format - #PENDING: get_report should return file or string, check for result.is_file instead of format - if format=="application/x-yaml" or format=="application/rdf+xml" - report + #QMRF-STUB + if params[:type] == Reports::ReportFactory::RT_QMRF + #raise Reports::BadRequest.new("only 'application/qmrf-xml' provided so far") if accept_header != "application/qmrf-xml" + content_type "application/qmrf-xml" + result = body(OpenTox::RestClientWrapper.get("http://ecb.jrc.ec.europa.eu/qsar/qsar-tools/qrf/QMRF_v1.2_FishTox.xml")) else - result = body(File.new(report)) + report = rs.get_report(params[:type],params[:id],accept_header) + format = Reports::ReportFormat.get_format(accept_header) + content_type format + #PENDING: get_report should return file or string, check for result.is_file instead of format + if format=="application/x-yaml" or format=="application/rdf+xml" + report + else + result = body(File.new(report)) + end end end end @@ -99,3 +106,17 @@ post '/report/:type' do end halt 202,task_uri end + + +post '/report/:type/:id' do + perform do |rs| + #QMRF-STUB + if params[:type] == Reports::ReportFactory::RT_QMRF + #raise Reports::BadRequest.new("only 'application/qmrf-xml' provided so far") if request.content_type != "application/qmrf-xml" + input = request.env["rack.input"].read + "save qmrf would have been successfull, received data with "+input.to_s.size.to_s+" characters, this is just a stub, changes discarded" + else + "operation not supported yet" + end + end +end diff --git a/report/report_factory.rb b/report/report_factory.rb index f62803d..646ecfd 100644 --- a/report/report_factory.rb +++ b/report/report_factory.rb @@ -17,12 +17,12 @@ VAL_ATTR_BAR_PLOT_REGR = [ :root_mean_squared_error, :mean_absolute_error, :r_sq # module Reports::ReportFactory - RT_FASTTOX = "fasttox" RT_VALIDATION = "validation" RT_CV = "crossvalidation" RT_ALG_COMP = "algorithm_comparison" + RT_QMRF = "qmrf" - REPORT_TYPES = [RT_VALIDATION, RT_CV, RT_ALG_COMP ] #,RT_FASTTOX + REPORT_TYPES = [RT_VALIDATION, RT_CV, RT_ALG_COMP, RT_QMRF ] # creates a report of a certain type according to the validation data in validation_set # @@ -31,8 +31,6 @@ module Reports::ReportFactory # def self.create_report(type, validation_set) case type - when RT_FASTTOX - raise "not yet implemented" when RT_VALIDATION create_report_validation(validation_set) when RT_CV @@ -40,7 +38,7 @@ module Reports::ReportFactory when RT_ALG_COMP create_report_compare_algorithms(validation_set) else - raise "unknown report type" + raise "unknown report type "+type.to_s end end diff --git a/report/report_persistance.rb b/report/report_persistance.rb index a06dba8..46a014e 100644 --- a/report/report_persistance.rb +++ b/report/report_persistance.rb @@ -1,4 +1,6 @@ +REPORT_DIR = File.join(Dir.pwd,'/reports') + # = Reports::ReportPersistance # # service that stores reports (Reports::ReportConent), and provides access in various formats @@ -68,10 +70,9 @@ class Reports::FileReportPersistance < Reports::ReportPersistance def initialize() raise "pls specify report-directory (:reports -> :report_dir) in config file" unless @@config[:reports] and @@config[:reports][:report_dir] - @report_dir = @@config[:reports][:report_dir] - FileUtils.mkdir @report_dir.to_s unless File.directory?(@report_dir) - raise "report cannot be found nor created" unless File.directory?(@report_dir) - LOGGER.debug "reports are stored in "+@report_dir.to_s + FileUtils.mkdir REPORT_DIR unless File.directory?(REPORT_DIR) + raise "report cannot be found nor created" unless File.directory?(REPORT_DIR) + LOGGER.debug "reports are stored in "+REPORT_DIR end def list_reports(type, filter_params=nil) @@ -165,7 +166,7 @@ class Reports::FileReportPersistance < Reports::ReportPersistance end def type_directory(type) - dir = @report_dir+"/"+type + dir = REPORT_DIR+"/"+type FileUtils.mkdir dir.to_s unless (File.directory?(dir)) return dir end @@ -230,6 +231,9 @@ module Reports end def list_reports(type, filter_params=nil) + #QMRF-STUB + return "1" if type == ReportFactory::RT_QMRF + filter_params = {} unless filter_params filter_params.each{ |k,v| raise Reports::BadRequest.new("no report-attribute: "+k.to_s) unless ReportData.column_names.include?(k.gsub(/_like$/,"")) } filter_params[:report_type] = type diff --git a/report/report_test.rb b/report/report_test.rb index ff4e3db..1c838bf 100644 --- a/report/report_test.rb +++ b/report/report_test.rb @@ -16,9 +16,15 @@ class Reports::ApplicationTest < Test::Unit::TestCase end def test_nothing - - get "/report/validation/1",nil,'HTTP_ACCEPT' => "application/rdf+xml"#"application/x-yaml" - puts last_response.body.to_s + + file = File.new("qmrf-report.xml") + raise "File not found: "+file.path.to_s unless File.exist?(file.path) + data = File.read(file.path) + puts OpenTox::RestClientWrapper.post("http://localhost/validation/report/qmrf/1",{:content_type => "application/qmrf-xml"},data).to_s.chomp + + #get "/report/qmrf/1",nil,'HTTP_ACCEPT' => "application/qmrf-xml"#"application/rdf+xml"#"application/x-yaml" + #get "/report/validation/1",nil,'HTTP_ACCEPT' => "application/rdf+xml"#"application/x-yaml" + #puts last_response.body.to_s #Reports::XMLReport.generate_demo_xml_report.write_to #raise "stop" diff --git a/report/validation_data.rb b/report/validation_data.rb index d87e491..0a25e87 100644 --- a/report/validation_data.rb +++ b/report/validation_data.rb @@ -19,7 +19,13 @@ end class Object def to_nice_s - return "%.2f" % self if is_a?(Float) + if is_a?(Float) + if self>0.01 + return "%.2f" % self + else + return self.to_s + end + end return collect{ |i| i.to_nice_s }.join(", ") if is_a?(Array) return collect{ |i,j| i.to_nice_s+": "+j.to_nice_s }.join(", ") if is_a?(Hash) return to_s diff --git a/test/test_examples.rb b/test/test_examples.rb index beeaa30..bbe6ddd 100644 --- a/test/test_examples.rb +++ b/test/test_examples.rb @@ -9,6 +9,53 @@ end module ValidationExamples + class IrisCrossvalidation < CrossValidation + def initialize + @dataset_file = File.new("data/IRIS_unitrisk.yaml","r") + @prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" + @num_folds = 10 + end + end + + class MajorityIrisCrossvalidation < IrisCrossvalidation + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + + class LazarIrisCrossvalidation < IrisCrossvalidation + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + ######################################################################################################## + + class IrisSplit < SplitTestValidation + def initialize + @dataset_file = File.new("data/IRIS_unitrisk.yaml","r") + @prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" + end + end + + class LazarIrisSplit < IrisSplit + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") + @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") + super + end + end + + class MajorityIrisSplit < IrisSplit + def initialize + @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/regr/algorithm") + super + end + end + ######################################################################################################## class HamsterSplit < SplitTestValidation @@ -86,21 +133,21 @@ module ValidationExamples ######################################################################################################## - class ISTHamsterCrossvalidationInsilico < CrossValidation + class ISTHamsterCrossvalidation < CrossValidation def initialize @dataset_uri = "http://webservices.in-silico.ch/dataset/108" @prediction_feature = "http://toxcreate.org/feature#Hamster%20Carcinogenicity%20(DSSTOX/CPDB)" end end - class MajorityISTHamsterCrossvalidation < ISTHamsterCrossvalidationInsilico + class MajorityISTHamsterCrossvalidation < ISTHamsterCrossvalidation def initialize @algorithm_uri = File.join(@@config[:services]["opentox-majority"],"/class/algorithm") super end end - class LazarISTHamsterCrossvalidation < ISTHamsterCrossvalidationInsilico + class LazarISTHamsterCrossvalidation < ISTHamsterCrossvalidation def initialize @algorithm_uri = File.join(@@config[:services]["opentox-algorithm"],"lazar") @algorithm_params = "feature_generation_uri="+File.join(@@config[:services]["opentox-algorithm"],"fminer") @@ -108,14 +155,47 @@ module ValidationExamples end end - class ISTLazarISTHamsterCrossvalidation < ISTHamsterCrossvalidationInsilico + class ISTLazarISTHamsterCrossvalidation < ISTHamsterCrossvalidation def initialize @algorithm_uri = "http://webservices.in-silico.ch/algorithm/lazar" @algorithm_params = "feature_generation_uri=http://webservices.in-silico.ch/algorithm/fminer" super end end + + ######################################################################################################## + + class ISTIrisCrossvalidation < CrossValidation + def initialize + @dataset_uri = "http://ot-dev.in-silico.ch/dataset/39" + @prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk" + end + end + + class ISTLazarISTIrisCrossvalidation < ISTIrisCrossvalidation + def initialize + @algorithm_uri = "http://ot-dev.in-silico.ch/algorithm/lazar" + @algorithm_params = "feature_generation_uri=http://ot-dev.in-silico.ch/algorithm/fminer" + super + end + end + + ######################################################################################################## + class ISTEpaCrossvalidation < CrossValidation + def initialize + @dataset_uri = "http://ot-dev.in-silico.ch/dataset/69" + @prediction_feature = "http://ot-dev.in-silico.ch/toxcreate/feature#EPA%20FHM" + end + end + + class ISTLazarISTEpaCrossvalidation < ISTEpaCrossvalidation + def initialize + @algorithm_uri = "http://ot-dev.in-silico.ch/algorithm/lazar" + @algorithm_params = "feature_generation_uri=http://ot-dev.in-silico.ch/algorithm/fminer" + super + end + end ######################################################################################################## @@ -192,6 +272,15 @@ module ValidationExamples "6b" => [ MLR_NTUA_CacoTrainingTest ], "6c" => [ MLR_NTUA2_CacoTrainingTest ], "6d" => [ MajorityCacoTrainingTest ], + + "7a" => [ LazarIrisSplit ], + "7b" => [ MajorityIrisSplit ], + + "8b" => [ MajorityIrisCrossvalidation ], + + "9a" => [ ISTLazarISTIrisCrossvalidation ], + + "10a" => [ ISTLazarISTEpaCrossvalidation ], } def self.list diff --git a/test/test_examples_util.rb b/test/test_examples_util.rb index 582b814..09f7d6c 100644 --- a/test/test_examples_util.rb +++ b/test/test_examples_util.rb @@ -114,11 +114,15 @@ module ValidationExamples assert_int_equal(predictions, num_instances-num_unpredicted) else regr_stats = val["regression_statistics".to_sym] - assert regr_stats!=nil + assert_not_nil regr_stats end end - private + private + def self.assert_not_nil(val,msg_suffix=nil) + raise msg_suffix.to_s+" is nil" if val==nil + end + def self.assert_int_equal(val1,val2,msg_suffix=nil) raise msg_suffix.to_s+" not equal: "+val1.to_s+" != "+val2.to_s unless val1==val2 end diff --git a/validation/validation_application.rb b/validation/validation_application.rb index cadfa43..a43a2a6 100644 --- a/validation/validation_application.rb +++ b/validation/validation_application.rb @@ -1,5 +1,5 @@ -[ 'rubygems', 'sinatra', 'sinatra/url_for', 'opentox-ruby-api-wrapper', 'logger', 'active_record', 'ar-extensions' ].each do |lib| +[ 'rubygems', 'sinatra', 'sinatra/url_for', 'active_record', 'ar-extensions', 'opentox-ruby-api-wrapper' ].each do |lib| require lib end @@ -200,16 +200,28 @@ post '/plain_training_test_split' do result[:training_dataset_uri]+"\n"+result[:test_dataset_uri]+"\n" end -post '/create_validation' do +post '/validate_datasets' do content_type "text/uri-list" task_uri = OpenTox::Task.as_task do - LOGGER.info "creating validation "+params.inspect + LOGGER.info "validating values "+params.inspect halt 400, "test_dataset_uri missing" unless params[:test_dataset_uri] halt 400, "prediction_datset_uri missing" unless params[:prediction_dataset_uri] - halt 400, "model_uri missing" unless params[:model_uri] - v = Validation::Validation.new params - v.compute_validation_stats() + if params[:model_uri] + v = Validation::Validation.new params + v.compute_validation_stats_with_model() + else + halt 400, "please specify 'model_uri' or 'prediction_feature'" unless params[:prediction_feature] + halt 400, "please specify 'model_uri' or 'predicted_feature'" unless params[:predicted_feature] + halt 400, "please specify 'model_uri' or set either 'classification' or 'regression' flag" unless + params[:classification] or params[:regression] + + predicted_feature = params.delete("predicted_feature") + clazz = params.delete("classification")!=nil + regr = params.delete("regression")!=nil + v = Validation::Validation.new params + v.compute_validation_stats((clazz and !regr),predicted_feature) + end content_type "text/uri-list" v.validation_uri end diff --git a/validation/validation_format.rb b/validation/validation_format.rb index 79526a9..a172f8a 100644 --- a/validation/validation_format.rb +++ b/validation/validation_format.rb @@ -74,16 +74,10 @@ module Validation LITERALS = [ :created_at, :real_runtime, :num_instances, :num_without_class, :percent_without_class, :num_unpredicted, :percent_unpredicted, - :crossvalidation_fold, #:crossvalidation_id, - :num_correct, :num_incorrect, :percent_correct, :percent_incorrect, - :area_under_roc, :false_negative_rate, :false_positive_rate, - :f_measure, :num_false_positives, :num_false_negatives, - :num_true_positives, :num_true_negatives, :precision, - :recall, :true_negative_rate, :true_positive_rate, - :confusion_matrix_value, :weighted_area_under_roc, - :target_variance_actual, :root_mean_squared_error, - :target_variance_predicted, :mean_absolute_error, :r_square, :class_value, - :confusion_matrix_actual, :confusion_matrix_predicted ] + :crossvalidation_fold ] + + (Lib::VAL_CLASS_PROPS - [ :confusion_matrix ]) + Lib::VAL_REGR_PROPS + + [ :class_value, :confusion_matrix_value, + :confusion_matrix_actual, :confusion_matrix_predicted ] LITERAL_NAMES = {:created_at => OT["date"] } diff --git a/validation/validation_service.rb b/validation/validation_service.rb index 96fd84a..cfbb681 100644 --- a/validation/validation_service.rb +++ b/validation/validation_service.rb @@ -39,10 +39,10 @@ module Validation $sinatra.halt 500,"do not set id manually" if params[:id] $sinatra.halt 500,"do not set uri manually" if params[:validation_uri] super params - self.save + self.save! raise "internal error, validation-id not set "+to_yaml if self.id==nil self.attributes = { :validation_uri => $sinatra.url_for("/"+self.id.to_s, :full).to_s } - self.save + self.save! end # deletes a validation @@ -79,7 +79,7 @@ module Validation model = OpenTox::Model::PredictionModel.build(algorithm_uri, params) $sinatra.halt 500,"model building failed" unless model self.attributes = { :model_uri => model.uri } - self.save + self.save! $sinatra.halt 500,"error after building model: model.dependent_variable != validation.prediciton_feature ("+ model.dependentVariables.to_s+" != "+self.prediction_feature+")" if self.prediction_feature!=model.dependentVariables @@ -98,7 +98,7 @@ module Validation unless self.algorithm_uri self.attributes = { :algorithm_uri => model.algorithm } - self.save + self.save! end if self.prediction_feature @@ -107,7 +107,7 @@ module Validation else $sinatra.halt 400, "model has no dependentVariables specified, please give prediction feature for model validation" unless model.dependentVariables self.attributes = { :prediction_feature => model.dependentVariables } - self.save + self.save! end prediction_dataset_uri = "" @@ -116,24 +116,30 @@ module Validation end self.attributes = { :prediction_dataset_uri => prediction_dataset_uri, :real_runtime => benchmark.real } - self.save + self.save! - compute_validation_stats(model) + compute_validation_stats_with_model( model ) end - - def compute_validation_stats(model = nil) - model = OpenTox::Model::PredictionModel.find(self.model_uri) unless model + def compute_validation_stats_with_model( model=nil ) + + model = OpenTox::Model::PredictionModel.find(self.model_uri) if model==nil and self.model_uri $sinatra.halt 400, "model not found: "+self.model_uri.to_s unless model + prediction_feature = self.prediction_feature ? nil : model.dependentVariables + algorithm_uri = self.algorithm_uri ? nil : model.algorithm + compute_validation_stats( model.classification?, model.predictedVariables, prediction_feature, algorithm_uri ) + end - self.attributes = { :prediction_feature => model.dependentVariables } unless self.prediction_feature - self.attributes = { :algorithm_uri => model.algorithm } unless self.algorithm_uri - self.save + def compute_validation_stats( classification, predicted_feature, prediction_feature=nil, algorithm_uri=nil) + + self.attributes = { :prediction_feature => prediction_feature } if self.prediction_feature==nil && prediction_feature + self.attributes = { :algorithm_uri => algorithm_uri } if self.algorithm_uri==nil && algorithm_uri + self.save! LOGGER.debug "computing prediction stats" - prediction = Lib::OTPredictions.new( model.classification?, + prediction = Lib::OTPredictions.new( classification, self.test_dataset_uri, self.test_target_dataset_uri, self.prediction_feature, - self.prediction_dataset_uri, model.predictedVariables ) + self.prediction_dataset_uri, predicted_feature ) if prediction.classification? self.attributes = { :classification_statistics => prediction.compute_stats } else @@ -145,7 +151,7 @@ module Validation :percent_without_class => prediction.percent_without_class, :num_unpredicted => prediction.num_unpredicted, :percent_unpredicted => prediction.percent_unpredicted } - self.save + self.save! end end @@ -161,10 +167,10 @@ module Validation params[:random_seed] = 1 if params[:random_seed]==nil params[:stratified] = false if params[:stratified]==nil super params - self.save + self.save! raise "internal error, crossvalidation-id not set" if self.id==nil self.attributes = { :crossvalidation_uri => $sinatra.url_for("/crossvalidation/"+self.id.to_s, :full) } - self.save + self.save! end # deletes a crossvalidation, all validations are deleted as well @@ -185,8 +191,9 @@ module Validation def perform_cv ( algorithm_params=nil ) LOGGER.debug "perform cv validations" - Validation.find( :all, :conditions => { :crossvalidation_id => id } ).each do |v| - v.validate_algorithm( algorithm_params ) + @tmp_validations.each do | val | + validation = Validation.new val + validation.validate_algorithm( algorithm_params ) #break end end @@ -274,6 +281,8 @@ module Validation test_features = orig_dataset.features.dclone - [prediction_feature] + @tmp_validations = [] + (1..self.num_folds).each do |n| datasetname = 'cv'+self.id.to_s + @@ -305,12 +314,13 @@ module Validation LOGGER.debug "test set: "+datasetname+"_test, compounds: "+test_compounds.size.to_s test_dataset_uri = orig_dataset.create_new_dataset( test_compounds, test_features, datasetname + '_test', source ) - validation = Validation.new :training_dataset_uri => train_dataset_uri, - :test_dataset_uri => test_dataset_uri, - :test_target_dataset_uri => self.dataset_uri, - :crossvalidation_id => self.id, :crossvalidation_fold => n, - :prediction_feature => prediction_feature, - :algorithm_uri => self.algorithm_uri + tmp_validation = { :training_dataset_uri => train_dataset_uri, + :test_dataset_uri => test_dataset_uri, + :test_target_dataset_uri => self.dataset_uri, + :crossvalidation_id => self.id, :crossvalidation_fold => n, + :prediction_feature => prediction_feature, + :algorithm_uri => self.algorithm_uri } + @tmp_validations << tmp_validation end end end diff --git a/validation/validation_test.rb b/validation/validation_test.rb index ce53c99..cbaa84b 100644 --- a/validation/validation_test.rb +++ b/validation/validation_test.rb @@ -1,4 +1,3 @@ -require "logger" require "uri" require "yaml" ENV['RACK_ENV'] = 'test' @@ -7,16 +6,20 @@ require 'test/unit' require 'rack/test' require 'lib/test_util.rb' require 'test/test_examples.rb' + LOGGER = MyLogger.new(STDOUT) LOGGER.datetime_format = "%Y-%m-%d %H:%M:%S " +LOGGER.formatter = Logger::Formatter.new + class ValidationTest < Test::Unit::TestCase include Rack::Test::Methods include Lib::TestUtil def test_it - #$test_case = self + $test_case = self + #get "/crossvalidation/4/statistics" # post "",:model_uri=>"http://localhost/model/1",:test_dataset_uri=>"http://localhost/dataset/3", # :test_target_dataset_uri=>"http://localhost/dataset/1" @@ -26,13 +29,26 @@ class ValidationTest < Test::Unit::TestCase # post "/test_validation",:select=>"6d" #,:report=>"yes,please" # puts last_response.body - #run_test("1b") #, "http://localhost/validation/321") - #run_test("3b", "http://localhost/validation/crossvalidation/1") +# post "/validate_datasets",{ +# :test_dataset_uri=>"http://localhost/dataset/204", +# :prediction_dataset_uri=>"http://localhost/dataset/206", +# :test_target_dataset_uri=>"http://localhost/dataset/202", +# :prediction_feature=>"http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk", +# :predicted_feature=>"http://ot-dev.in-silico.ch/toxcreate/feature#IRIS%20unit%20risk_lazar_regression", +# :regression=>"true"} +# #:classification=>"true"} +# puts last_response.body + + run_test("3b" ) #, "http://localhost/validation/826") #,"http://localhost/validation/report/validation/36") #, "http://localhost/validation/321") + + #run_test("7a") #,"http://localhost/validation/crossvalidation/10") #, "http://localhost/validation/321") + + #run_test("8b", "http://localhost/validation/crossvalidation/4") #puts Nightly.build_nightly("1") #prepare_examples - do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE + #do_test_examples # USES CURL, DO NOT FORGET TO RESTART VALIDATION SERVICE end def app |