diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ca0b0259912379bdc7b88371bebdb3e6f247a14c..fe69f418c2a635cc08769bed2ad92b14023b167e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -48,7 +48,9 @@ repos:
^src/ptbench/data/nih_cxr14_re/default.json|
^src/ptbench/data/padchest/idiap.json|
^src/ptbench/data/padchest/no_tb_idiap.json|
- ^tests/data/16bits.png
+ ^src/ptbench/data/padchest/no_tb_idiap.json|
+ ^tests/data/16bits.png|
+ ^doc/results/img/rad_sign_drop.png
)
- id: check-toml
- id: check-yaml
diff --git a/doc/index.rst b/doc/index.rst
index 7757bedbfbab012fae479aa3fcef0abd9bc88285..964e4f35d4916227ca78b731b1d87acc3d1d89d3 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -49,7 +49,8 @@ User Guide
:maxdepth: 2
install
- usage
+ usage/index
+ results/index
references
cli
config
diff --git a/doc/results/img/compare_densenet_mc_2000.jpg b/doc/results/img/compare_densenet_mc_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9d6de5eb81b6a0b1b916c9d6406ad56b948b2637
Binary files /dev/null and b/doc/results/img/compare_densenet_mc_2000.jpg differ
diff --git a/doc/results/img/compare_densenet_mc_ch_200.jpg b/doc/results/img/compare_densenet_mc_ch_200.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d1aafe50c618af8b55464a66c0a5ecfbf95aa185
Binary files /dev/null and b/doc/results/img/compare_densenet_mc_ch_200.jpg differ
diff --git a/doc/results/img/compare_densenet_mc_ch_in_150.jpg b/doc/results/img/compare_densenet_mc_ch_in_150.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f53fcf33360e2aba5173c4dbb21a90a9418a54b6
Binary files /dev/null and b/doc/results/img/compare_densenet_mc_ch_in_150.jpg differ
diff --git a/doc/results/img/compare_densenet_mc_ch_in_kfold_2000.jpg b/doc/results/img/compare_densenet_mc_ch_in_kfold_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..8bb7383c3599801085c125c066e208f82d3d2398
Binary files /dev/null and b/doc/results/img/compare_densenet_mc_ch_in_kfold_2000.jpg differ
diff --git a/doc/results/img/compare_densenet_mc_ch_in_norm_2000.jpg b/doc/results/img/compare_densenet_mc_ch_in_norm_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..805fcd9074cda417d6254d48f8048b4007a931b0
Binary files /dev/null and b/doc/results/img/compare_densenet_mc_ch_in_norm_2000.jpg differ
diff --git a/doc/results/img/compare_densenet_mc_ch_kfold_2000.jpg b/doc/results/img/compare_densenet_mc_ch_kfold_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7760c122dedf7145f75578d309c0322f48ed5b1a
Binary files /dev/null and b/doc/results/img/compare_densenet_mc_ch_kfold_2000.jpg differ
diff --git a/doc/results/img/compare_densenet_mc_ch_norm_2000.jpg b/doc/results/img/compare_densenet_mc_ch_norm_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..bde15997d0460945722dba693fd42ced5235c6b3
Binary files /dev/null and b/doc/results/img/compare_densenet_mc_ch_norm_2000.jpg differ
diff --git a/doc/results/img/compare_densenet_mc_kfold_2000.jpg b/doc/results/img/compare_densenet_mc_kfold_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..4e88c2622c57e5405146312213320406b13937dc
Binary files /dev/null and b/doc/results/img/compare_densenet_mc_kfold_2000.jpg differ
diff --git a/doc/results/img/compare_densenet_mc_norm_2000.jpg b/doc/results/img/compare_densenet_mc_norm_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9b6a816f7009650685a5a73f2e6d52442d8aea4e
Binary files /dev/null and b/doc/results/img/compare_densenet_mc_norm_2000.jpg differ
diff --git a/doc/results/img/compare_densenetpreIN_mc_ch_ch_kfold_600.jpg b/doc/results/img/compare_densenetpreIN_mc_ch_ch_kfold_600.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f6d266ec6162e7cedc7a70190cb359bc484a69ed
Binary files /dev/null and b/doc/results/img/compare_densenetpreIN_mc_ch_ch_kfold_600.jpg differ
diff --git a/doc/results/img/compare_densenetpreIN_mc_ch_kfold_600.jpg b/doc/results/img/compare_densenetpreIN_mc_ch_kfold_600.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f85d87881ec1c9ff0d3b305f393e9eeb51b5ea05
Binary files /dev/null and b/doc/results/img/compare_densenetpreIN_mc_ch_kfold_600.jpg differ
diff --git a/doc/results/img/compare_densenetpreIN_mc_kfold_600.jpg b/doc/results/img/compare_densenetpreIN_mc_kfold_600.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..19d2628afb133cf1f4f47e9e0a3bf741d634edda
Binary files /dev/null and b/doc/results/img/compare_densenetpreIN_mc_kfold_600.jpg differ
diff --git a/doc/results/img/compare_densenetpre_mc_ch_in_kfold_300.jpg b/doc/results/img/compare_densenetpre_mc_ch_in_kfold_300.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d879e8aa458e5badedcea6f338da0981ca72e439
Binary files /dev/null and b/doc/results/img/compare_densenetpre_mc_ch_in_kfold_300.jpg differ
diff --git a/doc/results/img/compare_densenetpre_mc_ch_kfold_300.jpg b/doc/results/img/compare_densenetpre_mc_ch_kfold_300.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b84a15488f893072abdbc0c5083aefa1e116707f
Binary files /dev/null and b/doc/results/img/compare_densenetpre_mc_ch_kfold_300.jpg differ
diff --git a/doc/results/img/compare_densenetpre_mc_kfold_300.jpg b/doc/results/img/compare_densenetpre_mc_kfold_300.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7c24047aec7239d1ead13ebca19bdcc84178ff6a
Binary files /dev/null and b/doc/results/img/compare_densenetpre_mc_kfold_300.jpg differ
diff --git a/doc/results/img/compare_densenetprenih0001b4_mc_200.jpg b/doc/results/img/compare_densenetprenih0001b4_mc_200.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..762de49ea85ed0476b0591ecbfb52aaeb3b29969
Binary files /dev/null and b/doc/results/img/compare_densenetprenih0001b4_mc_200.jpg differ
diff --git a/doc/results/img/compare_densenetprenih0001b4_mc_ch_200.jpg b/doc/results/img/compare_densenetprenih0001b4_mc_ch_200.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..79dbad2610212a95c9bbd7d318aa062fcd64326f
Binary files /dev/null and b/doc/results/img/compare_densenetprenih0001b4_mc_ch_200.jpg differ
diff --git a/doc/results/img/compare_densenetprenih0001b4_mc_ch_in_200.jpg b/doc/results/img/compare_densenetprenih0001b4_mc_ch_in_200.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..eb0bd2a61763a3ef752c7dbdaed7ad20e51f007b
Binary files /dev/null and b/doc/results/img/compare_densenetprenih0001b4_mc_ch_in_200.jpg differ
diff --git a/doc/results/img/compare_densenetprenih0001b8_mc_300.jpg b/doc/results/img/compare_densenetprenih0001b8_mc_300.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..fa1947252f09cfe9e120b6254c69dea3eaffbfc6
Binary files /dev/null and b/doc/results/img/compare_densenetprenih0001b8_mc_300.jpg differ
diff --git a/doc/results/img/compare_densenetprenih0001b8_mc_ch_300.jpg b/doc/results/img/compare_densenetprenih0001b8_mc_ch_300.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..afd9a26d72b0e143a6e847167c501f3da7a91914
Binary files /dev/null and b/doc/results/img/compare_densenetprenih0001b8_mc_ch_300.jpg differ
diff --git a/doc/results/img/compare_densenetprenih0001b8_mc_ch_in_300.jpg b/doc/results/img/compare_densenetprenih0001b8_mc_ch_in_300.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b2935a3d13587a7f63a2bd789d2d706fb6af6baa
Binary files /dev/null and b/doc/results/img/compare_densenetprenih0001b8_mc_ch_in_300.jpg differ
diff --git a/doc/results/img/compare_logreg_mc_100.jpg b/doc/results/img/compare_logreg_mc_100.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..af439b82cb52398bdc907a13a0cfdb1f8f28885b
Binary files /dev/null and b/doc/results/img/compare_logreg_mc_100.jpg differ
diff --git a/doc/results/img/compare_logreg_mc_ch_100.jpg b/doc/results/img/compare_logreg_mc_ch_100.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..61ff8dd7dc61823b3f2069083ceb4ee0119eb8bf
Binary files /dev/null and b/doc/results/img/compare_logreg_mc_ch_100.jpg differ
diff --git a/doc/results/img/compare_logreg_mc_ch_in_100.jpg b/doc/results/img/compare_logreg_mc_ch_in_100.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..fb60eb078a75ffef6596037c31da493aae9504fd
Binary files /dev/null and b/doc/results/img/compare_logreg_mc_ch_in_100.jpg differ
diff --git a/doc/results/img/compare_logreg_mc_ch_in_kfold_100.jpg b/doc/results/img/compare_logreg_mc_ch_in_kfold_100.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b82ab31749a3f7ffa8f3b33fd27dbc79e3fe420d
Binary files /dev/null and b/doc/results/img/compare_logreg_mc_ch_in_kfold_100.jpg differ
diff --git a/doc/results/img/compare_logreg_mc_ch_kfold_100.jpg b/doc/results/img/compare_logreg_mc_ch_kfold_100.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..311a6cb08fb7e69a9668b0171acc55191a773de8
Binary files /dev/null and b/doc/results/img/compare_logreg_mc_ch_kfold_100.jpg differ
diff --git a/doc/results/img/compare_logreg_mc_kfold_150.jpg b/doc/results/img/compare_logreg_mc_kfold_150.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..669ab99f40a84475b60eafca92015cc40b710a28
Binary files /dev/null and b/doc/results/img/compare_logreg_mc_kfold_150.jpg differ
diff --git a/doc/results/img/compare_pasa_mc_2000.jpg b/doc/results/img/compare_pasa_mc_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..fb0d57bea5cca74532969e83c3685d1a93e49aa8
Binary files /dev/null and b/doc/results/img/compare_pasa_mc_2000.jpg differ
diff --git a/doc/results/img/compare_pasa_mc_ch_2000.jpg b/doc/results/img/compare_pasa_mc_ch_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ee8637e4062db94adb13531acbabd6bf4e02c3e8
Binary files /dev/null and b/doc/results/img/compare_pasa_mc_ch_2000.jpg differ
diff --git a/doc/results/img/compare_pasa_mc_ch_in_2000.jpg b/doc/results/img/compare_pasa_mc_ch_in_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ce3d99ff506664a965646fcfef9e591d585db1fd
Binary files /dev/null and b/doc/results/img/compare_pasa_mc_ch_in_2000.jpg differ
diff --git a/doc/results/img/compare_pasa_mc_ch_in_kfold_500.jpg b/doc/results/img/compare_pasa_mc_ch_in_kfold_500.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..772bd29855fccdeb53039c2c3b675510f0ef3ab8
Binary files /dev/null and b/doc/results/img/compare_pasa_mc_ch_in_kfold_500.jpg differ
diff --git a/doc/results/img/compare_pasa_mc_ch_in_norm_2000.jpg b/doc/results/img/compare_pasa_mc_ch_in_norm_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b1db6c6dbdfe710be53012a307637adbf83316d2
Binary files /dev/null and b/doc/results/img/compare_pasa_mc_ch_in_norm_2000.jpg differ
diff --git a/doc/results/img/compare_pasa_mc_ch_kfold_500.jpg b/doc/results/img/compare_pasa_mc_ch_kfold_500.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e112ce5119e60a74bef659e70e9a6e85801abe9e
Binary files /dev/null and b/doc/results/img/compare_pasa_mc_ch_kfold_500.jpg differ
diff --git a/doc/results/img/compare_pasa_mc_ch_norm_2000.jpg b/doc/results/img/compare_pasa_mc_ch_norm_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1d0564caf578b3f70f66fea3d86d83c6541c852d
Binary files /dev/null and b/doc/results/img/compare_pasa_mc_ch_norm_2000.jpg differ
diff --git a/doc/results/img/compare_pasa_mc_kfold_500.jpg b/doc/results/img/compare_pasa_mc_kfold_500.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d0e018bb9b615f2ca15aaba46c9302e1403b6807
Binary files /dev/null and b/doc/results/img/compare_pasa_mc_kfold_500.jpg differ
diff --git a/doc/results/img/compare_pasa_mc_norm_2000.jpg b/doc/results/img/compare_pasa_mc_norm_2000.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e5df3e8f373d467d28024a4b44a03bfea396751e
Binary files /dev/null and b/doc/results/img/compare_pasa_mc_norm_2000.jpg differ
diff --git a/doc/results/img/compare_signstotb_mc_50.jpg b/doc/results/img/compare_signstotb_mc_50.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..753a04da06ff93f9b8786f150dde8009d7874926
Binary files /dev/null and b/doc/results/img/compare_signstotb_mc_50.jpg differ
diff --git a/doc/results/img/compare_signstotb_mc_ch_50.jpg b/doc/results/img/compare_signstotb_mc_ch_50.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9a290e6c14c324f2169d61da575adb715c3c2ec5
Binary files /dev/null and b/doc/results/img/compare_signstotb_mc_ch_50.jpg differ
diff --git a/doc/results/img/compare_signstotb_mc_ch_in_50.jpg b/doc/results/img/compare_signstotb_mc_ch_in_50.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..8afb5776f2a83a12c558d2f410a26f0fc427b1ba
Binary files /dev/null and b/doc/results/img/compare_signstotb_mc_ch_in_50.jpg differ
diff --git a/doc/results/img/rad_sign_drop.png b/doc/results/img/rad_sign_drop.png
new file mode 100644
index 0000000000000000000000000000000000000000..70f5e9ab38e4e13a9e57c1762b8659b926dd2a8d
Binary files /dev/null and b/doc/results/img/rad_sign_drop.png differ
diff --git a/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_test.jpg b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_test.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0cb7af9cae4c7ee01d8fba2195178b1549722f59
Binary files /dev/null and b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_test.jpg differ
diff --git a/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_train.jpg b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_train.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..3e0b6677b6fbfbc0babf8d3e4757bcd0010b31ff
Binary files /dev/null and b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_train.jpg differ
diff --git a/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_validation.jpg b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_validation.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..38b9d373a2c299083583c81141ae36d6e577c87c
Binary files /dev/null and b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_validation.jpg differ
diff --git a/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_in_test.jpg b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_in_test.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..666d3ebd9592a8a8cf681b28ecef0fdaf254ac48
Binary files /dev/null and b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_in_test.jpg differ
diff --git a/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_in_train.jpg b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_in_train.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9b0a09c748d754623f9093b12b095e1621a88ce8
Binary files /dev/null and b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_in_train.jpg differ
diff --git a/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_in_validation.jpg b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_in_validation.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..2f472d76c6d74a2fa39631fd4ff89b5e89bea134
Binary files /dev/null and b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_in_validation.jpg differ
diff --git a/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_test.jpg b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_test.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1c3cdbf8610cca150334eada2c32b73bdd56adb0
Binary files /dev/null and b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_test.jpg differ
diff --git a/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_train.jpg b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_train.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..851fa818a890f491551f7ad459576edb1fffb3cb
Binary files /dev/null and b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_train.jpg differ
diff --git a/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_validation.jpg b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_validation.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7fa0c15569bfab1d820893e6fc480a4c31021b8f
Binary files /dev/null and b/doc/results/img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_validation.jpg differ
diff --git a/doc/results/index.rst b/doc/results/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..29ad8a10c4c54f3b8164fd04acab529f30e0ad29
--- /dev/null
+++ b/doc/results/index.rst
@@ -0,0 +1,390 @@
+.. SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+..
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+.. _ptbench.results:
+
+=========
+ Results
+=========
+
+This section summarizes results that can be obtained with this package.
+
+Models optimization
+-------------------
+
+In the link below, you will find information about the optimization of each
+model we used.
+
+.. toctree::
+ :maxdepth: 2
+
+ optimization/pasa
+ optimization/densenet
+ optimization/logreg
+ optimization/signstotb
+
+
+Models training runtime and memory footprint
+--------------------------------------------
+
+In the link below, you will find information about the training runtime and the
+memory footprint of each model we used.
+
+.. toctree::
+ :maxdepth: 2
+
+ runtime
+
+
+AUROC Scores
+------------
+
+* Benchmark results for models: Pasa, DenseNet, SignsToTB
+* Each dataset is split in a training, a validation and a testing subset
+* Datasets names are abbreviated as follows: Montgomery (MC), Shenzhen (CH),
+ Indian (IN)
+* Models are only trained on the training subset
+* During the training session, we keep checkpoints for the best performing
+ networks based on the validation set. The best performing network during
+ training is used for evaluation.
+* Model resource configuration links are linked to the originating
+ configuration files used to obtain these results.
+
+K-folding
+^^^^^^^^^
+
+Stratified k-folding has been used (10 folds) to generate these results.
+
+.. tip::
+
+ To generate the following results, you first need to predict TB on each
+ fold, then use the :ref:`aggregpred command <ptbench.cli>` to aggregate the
+ predictions together, and finally evaluate this new file using the
+ :ref:`compare command <ptbench.cli>`.
+
+Pasa and DenseNet-121 (random initialization)
+"""""""""""""""""""""""""""""""""""""""""""""
+
+Thresholds used:
+
+* Pasa trained on MC, test on MC, mean threshold: 0.5057
+* Pasa trained on MC-CH, test on MC-CH, mean threshold: 0.4966
+* Pasa trained on MC-CH-IN, test on MC-CH-IN, mean threshold: 0.4135
+* Densenet trained on MC, test on MC, mean threshold: 0.5183
+* Densenet trained on MC-CH, test on MC-CH, mean threshold: 0.2555
+* Densenet trained on MC-CH-IN, test on MC-CH-IN, mean threshold: 0.4037
+
+.. list-table::
+
+ * - AUC
+ - MC test
+ - CH test
+ - IN test
+ * - Pasa (train: MC)
+ - 0.890
+ - 0.576
+ - 0.642
+ * - Pasa (train: MC+CH)
+ - 0.870
+ - 0.893
+ - 0.669
+ * - Pasa (train: MC+CH+IN)
+ - 0.881
+ - 0.898
+ - 0.848
+ * - DenseNet-121 (train: MC)
+ - 0.822
+ - 0.607
+ - 0.625
+ * - DenseNet-121 (train: MC+CH)
+ - 0.883
+ - 0.905
+ - 0.672
+ * - DenseNet-121 (train: MC+CH+IN)
+ - 0.860
+ - 0.917
+ - 0.850
+
+.. list-table::
+
+ * - .. figure:: img/compare_pasa_mc_kfold_500.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for Pasa model trained on normalized-kfold MC
+
+ :py:mod:`Pasa <ptbench.configs.models.pasa>`: Pasa trained on normalized-kfold MC
+ - .. figure:: img/compare_pasa_mc_ch_kfold_500.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for Pasa model trained on normalized-kfold MC-CH
+
+ :py:mod:`Pasa <ptbench.configs.models.pasa>`: Pasa trained on normalized-kfold MC-CH
+ - .. figure:: img/compare_pasa_mc_ch_in_kfold_500.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for Pasa model trained on normalized-kfold MC-CH-IN
+
+ :py:mod:`Pasa <ptbench.configs.models.pasa>`: Pasa trained on normalized-kfold MC-CH-IN
+ * - .. figure:: img/compare_densenet_mc_kfold_2000.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for DenseNet model trained on normalized-kfold MC
+
+ :py:mod:`DenseNet <ptbench.configs.models.densenet>`: DenseNet trained on normalized-kfold MC
+ - .. figure:: img/compare_densenet_mc_ch_kfold_2000.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for DenseNet model trained on normalized-kfold MC-CH
+
+ :py:mod:`DenseNet <ptbench.configs.models.densenet>`: DenseNet trained on normalized-kfold MC-CH
+ - .. figure:: img/compare_densenet_mc_ch_in_kfold_2000.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for DenseNet model trained on normalized-kfold MC-CH-IN
+
+ :py:mod:`DenseNet <ptbench.configs.models.densenet>`: DenseNet trained on normalized-kfold MC-CH-IN
+
+DenseNet-121 (pretrained on ImageNet)
+"""""""""""""""""""""""""""""""""""""
+
+Thresholds used:
+
+* DenseNet (pretrained on ImageNet) trained on MC, test on MC, mean threshold: 0.3581
+* DenseNet (pretrained on ImageNet) trained on MC-CH, test on MC-CH, mean threshold: 0.3319
+* DenseNet (pretrained on ImageNet) trained on MC-CH-IN, test on MC-CH-IN, mean threshold: 0.4048
+
+.. list-table::
+
+ * - AUC
+ - MC test
+ - CH test
+ - IN test
+ * - DenseNet-121 (train: MC)
+ - 0.910
+ - 0.814
+ - 0.817
+ * - DenseNet-121 (train: MC+CH)
+ - 0.948
+ - 0.946
+ - 0.816
+ * - DenseNet-121 (train: MC+CH+IN)
+ - 0.925
+ - 0.944
+ - 0.911
+
+.. list-table::
+
+ * - .. figure:: img/compare_densenetpreIN_mc_kfold_600.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for DenseNet model trained on normalized-kfold MC
+
+ :py:mod:`DenseNet <ptbench.configs.models.densenet>` DenseNet trained on normalized-kfold MC
+ - .. figure:: img/compare_densenetpreIN_mc_ch_kfold_600.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for DenseNet model trained on normalized-kfold MC-CH
+
+ :py:mod:`DenseNet <ptbench.configs.models.densenet>` DenseNet trained on normalized-kfold MC-CH
+ - .. figure:: img/compare_densenetpreIN_mc_ch_ch_kfold_600.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for DenseNet model trained on normalized-kfold MC-CH-IN
+
+ :py:mod:`DenseNet <ptbench.configs.models.densenet>` DenseNet trained on normalized-kfold MC-CH-IN
+
+Logistic Regression Classifier
+""""""""""""""""""""""""""""""
+
+Thresholds used:
+
+* LogReg trained on MC, test on MC, mean threshold: 0.534
+* LogReg trained on MC-CH, test on MC-CH, mean threshold: 0.2838
+* LogReg trained on MC-CH-IN, test on MC-CH-IN, mean threshold: 0.2371
+
+.. list-table::
+
+ * - AUC
+ - MC test
+ - CH test
+ - IN test
+ * - Indirect (train: MC)
+ - 0.966
+ - 0.867
+ - 0.926
+ * - Indirect (train: MC+CH)
+ - 0.961
+ - 0.901
+ - 0.928
+ * - Indirect (train: MC+CH+IN)
+ - 0.951
+ - 0.895
+ - 0.920
+
+.. list-table::
+
+ * - .. figure:: img/compare_logreg_mc_kfold_150.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for LogReg model trained on normalized-kfold MC
+
+ :py:mod:`LogReg <ptbench.configs.models.logistic_regression>`: LogReg trained on normalized-kfold MC
+ - .. figure:: img/compare_logreg_mc_ch_kfold_100.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for LogReg model trained on normalized-kfold MC-CH
+
+ :py:mod:`LogReg <ptbench.configs.models.logistic_regression>`: LogReg trained on normalized-kfold MC-CH
+ - .. figure:: img/compare_logreg_mc_ch_in_kfold_100.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for LogReg model trained on normalized-kfold MC-CH-IN
+
+ :py:mod:`LogReg <ptbench.configs.models.logistic_regression>`: LogReg trained on normalized-kfold MC-CH-IN
+
+DenseNet-121 (pretrained on ImageNet and NIH CXR14)
+"""""""""""""""""""""""""""""""""""""""""""""""""""
+
+Thresholds used:
+
+* DenseNetPre trained on MC, test on MC, mean threshold: 0.4126
+* DenseNetPre trained on MC-CH, test on MC-CH, mean threshold: 0.3711
+* DenseNetPre trained on MC-CH-IN, test on MC-CH-IN, mean threshold: 0.4255
+
+.. list-table::
+
+ * - AUC
+ - MC test
+ - CH test
+ - IN test
+ * - DenseNet-121 (train: MC)
+ - 0.966
+ - 0.917
+ - 0.901
+ * - DenseNet-121 (train: MC+CH)
+ - 0.984
+ - 0.979
+ - 0.869
+ * - DenseNet-121 (train: MC+CH+IN)
+ - 0.965
+ - 0.978
+ - 0.931
+
+.. list-table::
+
+ * - .. figure:: img/compare_densenetpre_mc_kfold_300.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for DenseNet model trained on normalized-kfold MC (pretrained on NIH)
+
+ :py:mod:`DenseNet <ptbench.configs.models.densenet>`: DenseNet trained on normalized-kfold MC (pretrained on NIH)
+ - .. figure:: img/compare_densenetpre_mc_ch_kfold_300.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for DenseNet model trained on normalized-kfold MC-CH (pretrained on NIH)
+
+ :py:mod:`DenseNet <ptbench.configs.models.densenet>`: DenseNet trained on normalized-kfold MC-CH (pretrained on NIH)
+ - .. figure:: img/compare_densenetpre_mc_ch_in_kfold_300.jpg
+ :align: center
+ :scale: 50%
+ :alt: Testing sets ROC curves for DenseNet model trained on normalized-kfold MC-CH-IN (pretrained on NIH)
+
+ :py:mod:`DenseNet <ptbench.configs.models.densenet>`: DenseNet trained on normalized-kfold MC-CH-IN (pretrained on NIH)
+
+
+Global sensitivity analysis (relevance)
+---------------------------------------
+
+Model used to generate the following figures: LogReg trained on MC-CH-IN fold 0 for 100 epochs.
+
+.. tip::
+
+ Use the ``--relevance-analysis`` argument of the :ref:`predict command
+ <ptbench.cli>` to generate the following plots.
+
+* Green color: likely TB
+* Orange color: Could be TB
+* Dark red color: Unlikely TB
+
+As CH is the largest dataset, its relevance analysis is computed on more images
+and is supposed to be more stable. Similarly, train sets are larger.
+We notice the systematic importance of Nodule, Pleural Thickening, Fibrosis,
+Mass, Consolidation and Pleural Effusion.
+
+.. list-table::
+
+ * - .. figure:: img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_train.jpg
+ :align: center
+ :scale: 50%
+ :alt: Relevance analysis on train MC
+
+ Relevance analysis on train MC
+ - .. figure:: img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_validation.jpg
+ :align: center
+ :scale: 50%
+ :alt: Relevance analysis on validation MC
+
+ Relevance analysis on validation MC
+ - .. figure:: img/relevance_analysis/logreg_mc_ch_in_f0_100_mc_test.jpg
+ :align: center
+ :scale: 50%
+ :alt: Relevance analysis on test MC
+
+ Relevance analysis on test MC
+ * - .. figure:: img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_train.jpg
+ :align: center
+ :scale: 50%
+ :alt: Relevance analysis on train CH
+
+ Relevance analysis on train CH
+ - .. figure:: img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_validation.jpg
+ :align: center
+ :scale: 50%
+ :alt: Relevance analysis on validation CH
+
+ Relevance analysis on validation CH
+ - .. figure:: img/relevance_analysis/logreg_mc_ch_in_f0_100_ch_test.jpg
+ :align: center
+ :scale: 50%
+ :alt: Relevance analysis on test CH
+
+ Relevance analysis on test CH
+ * - .. figure:: img/relevance_analysis/logreg_mc_ch_in_f0_100_in_train.jpg
+ :align: center
+ :scale: 50%
+ :alt: Relevance analysis on train IN
+
+ Relevance analysis on train IN
+ - .. figure:: img/relevance_analysis/logreg_mc_ch_in_f0_100_in_validation.jpg
+ :align: center
+ :scale: 50%
+ :alt: Relevance analysis on validation IN
+
+ Relevance analysis on validation IN
+ - .. figure:: img/relevance_analysis/logreg_mc_ch_in_f0_100_in_test.jpg
+ :align: center
+ :scale: 50%
+ :alt: Relevance analysis on test IN
+
+ Relevance analysis on test IN
+
+
+Ablation study
+--------------
+
+Here, we removed the data of each sign, one after the other, from the dataset
+for both model training and prediction. LogReg trained on MC-CH-IN fold 0 for
+100 epochs has been used to generate the following plot.
+
+Predictive capabilities of our logistic regression model after removing the
+data for each radiological sign (d0-d13 correspond, in this order, to
+cardiomegaly, emphysema, effusion, hernia, infiltration, mass, nodule,
+atelectasis, pneumothorax, pleural thickening, pneumonia, fibrosis, edema, and
+consolidation).
+
+- .. figure:: img/rad_sign_drop.png
+ :width: 400px
+
+
+.. include:: ../links.rst
diff --git a/doc/results/optimization/densenet.rst b/doc/results/optimization/densenet.rst
new file mode 100644
index 0000000000000000000000000000000000000000..d6a4c1a87898e9992760d306be76dc9d860c1673
--- /dev/null
+++ b/doc/results/optimization/densenet.rst
@@ -0,0 +1,136 @@
+.. SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+..
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+.. _ptbench.results.optimization.densenet:
+
+=============================
+ Densenet model optimization
+=============================
+
+.. note::
+
+ The Densenet121 model contains 7'216'513 parameters.
+
+Training on TB datasets from scratch
+------------------------------------
+
+To select the optimal learning rate and batch size for the training on the
+TB datasets from scratch (densenet not pretrained),
+we did a grid search with the following parameters.
+
+- learning rate of 1e-4, 5e-5 and 1e-5
+- batch size of 4 and 8
+
+We systematically used the training set of the combined dataset MC-CH-IN for
+this optimization.
+
+**The minimum validation loss we found is 0.3168 by using a learning rate of
+5e-5 and a batch size of 8**.
+
+
+Minimum validation loss grid search
+===================================
+
+This table indicates the minimum validation loss obtained for each combination
+of learning rate and batch size.
+
+.. list-table::
+
+ * - Learning rate
+ - Batch size of 4
+ - Batch size of 8
+ * - 1e-4 (training for 600 epochs)
+ - 0.3658
+ - 0.3676
+ * - 5e-5 (training for 150 epochs)
+ - 0.3490
+ - **0.3168**
+ * - 1e-5 (training for 1000 epochs)
+ - 0.3791
+ - 0.3831
+
+
+Thresholds selection
+====================
+
+The threshold was systematically selected on the validation set of the datasets
+on which the model was trained.
+
+- Threshold for Densenet trained on MC: 0.599
+- Threshold for Densenet trained on MC-CH: 0.519
+- Threshold for Densenet trained on MC-CH-IN: 0.472
+
+
+Pre-training on NIH CXR14
+-------------------------
+
+We used the pretrained Densenet121 model provided by PyTorch. For the
+pretraining on the NIH CXR14 dataset, the hyperparameters from the CheXNeXt
+study were used: batch size of 8, learning rate 1e-4 and the default Adam
+optimizer parameters: beta_1=0.9, beta_2=0.999, epsilon = 1e-8.
+
+
+Fine-tuning on TB datasets
+--------------------------
+
+To select the optimal learning rate and batch size for the fine-tuning (after
+the pre-training on NIH CXR14), we did a grid search with the following
+parameters.
+
+- learning rate of 1e-4, 1e-5, 5e-6, 1e-6
+- batch size of 4, 8 and 16
+
+We systematically used the training set of the combined dataset MC-CH-IN for
+this optimization.
+
+**The minimum validation loss we found is 0.1511 by using a learning rate of
+1e-4 and a batch size of 8**.
+
+
+Minimum validation loss grid search
+===================================
+
+This table indicates the minimum validation loss obtained for each combination
+of learning rate and batch size.
+
+.. list-table::
+
+ * - Learning rate
+ - Batch size of 4
+ - Batch size of 8
+ - Batch size of 16
+ * - 1e-4 (training for 300 epochs)
+ - 0.2053
+ - **0.1511**
+ - 0.2372
+ * - 1e-5 (training for 500 epochs)
+ - 0.1832
+ - 0.1931
+ - 0.2326
+ * - 5e-6 (training for 300 epochs)
+ - 0.1932
+ - 0.2234
+ - 0.2298
+ * - 1e-6 (training for 600 epochs)
+ - 0.2086
+ - 0.2139
+ - 0.2138
+
+
+Thresholds selection
+====================
+
+The threshold was systematically selected on the validation set of the datasets
+on which the model was trained.
+
+- Threshold for Densenet trained on MC: 0.688
+- Threshold for Densenet trained on MC-CH: 0.386
+- Threshold for Densenet trained on MC-CH-IN: 0.432
+
+
+Other hyperparameters
+=====================
+
+The default Adam optimizer parameters were used: beta_1=0.9, beta_2=0.999,
+epsilon = 1e-8.
diff --git a/doc/results/optimization/logreg.rst b/doc/results/optimization/logreg.rst
new file mode 100644
index 0000000000000000000000000000000000000000..cc8f8574bddbfdeb32eb14ad4ced8e3d457789a6
--- /dev/null
+++ b/doc/results/optimization/logreg.rst
@@ -0,0 +1,78 @@
+.. SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+..
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+.. _ptbench.results.optimization.logreg:
+
+===========================
+ LogReg model optimization
+===========================
+
+.. note::
+
+ The Logistic Regression model contains 15 parameters.
+
+
+LogReg is a logistic regression model created to predict TB presence based on
+the fourteen radiological signs predicted by the DensenetRS model. To train
+this model, we created new features for the Montgomery, Shenzhen and Indian
+dataset by predicting the presence of radiological signs on each of them with
+DensenetRS. Those new datasets versions can be identified by the _RS
+(for Radiological Signs) in their name.
+
+To select the optimal learning rate and the optimal number of neurons for the
+LogReg model, we did a grid search with the following parameters.
+
+- learning rate from 1e-1 to 1e-4
+- batch size of 4, 8 and 16
+
+We systematically used the training set of the combined dataset MC-CH-IN for
+this optimization.
+
+**The minimum validation loss we found is 0.3835 by using a learning rate of
+1e-2 and a batch size of 4**.
+
+
+Minimum validation loss grid search
+-----------------------------------
+
+.. list-table::
+
+ * - Learning rate
+ - Batch size of 4
+ - Batch size of 8
+ - Batch size of 16
+ * - 1e-1 (training for 50 epochs)
+ - 0.3932
+ - 0.4013
+ - 0.4229
+ * - 1e-2 (training for 100 epochs)
+ - **0.3835**
+ - 0.3998
+ - 0.4126
+ * - 1e-3 (training for 200 epochs)
+ - 0.3875
+ - 0.4075
+ - 0.4188
+ * - 1e-4 (training for 800 epochs)
+ - 0.3942
+ - 0.4059
+ - 0.4123
+
+
+Thresholds selection
+--------------------
+
+The threshold was systematically selected on the validation set of the datasets
+on which the model was trained.
+
+- Threshold for LogReg trained on MC: 0.568
+- Threshold for LogReg trained on MC-CH: 0.372
+- Threshold for LogReg trained on MC-CH-IN: 0.430
+
+
+Other hyperparameters
+=====================
+
+The default Adam optimizer parameters were used: beta_1=0.9, beta_2=0.999,
+epsilon = 1e-8.
diff --git a/doc/results/optimization/pasa.rst b/doc/results/optimization/pasa.rst
new file mode 100644
index 0000000000000000000000000000000000000000..901d99f63cb84eecca18b8ab687600b24b97979d
--- /dev/null
+++ b/doc/results/optimization/pasa.rst
@@ -0,0 +1,29 @@
+.. SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+..
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+.. _ptbench.results.optimization.pasa:
+
+=========================
+ Pasa model optimization
+=========================
+
+.. note::
+
+ The Pasa model contains 201'905 parameters.
+
+
+Model hyperparameters from the original study were used: batch size of 4,
+learning rate 8e-5 and the default Adam optimizer parameters: beta_1=0.9,
+beta_2=0.999, epsilon = 1e-8. The Pasa model has not been pretrained.
+
+
+Thresholds selection
+--------------------
+
+The threshold was systematically selected on the validation set of the datasets
+on which the model was trained.
+
+- Threshold for Pasa trained on MC: 0.577
+- Threshold for Pasa trained on MC-CH: 0.417
+- Threshold for Pasa trained on MC-CH-IN: 0.235
diff --git a/doc/results/optimization/signstotb.rst b/doc/results/optimization/signstotb.rst
new file mode 100644
index 0000000000000000000000000000000000000000..a7b100341ad2089743a66de37d713945ec5544a0
--- /dev/null
+++ b/doc/results/optimization/signstotb.rst
@@ -0,0 +1,74 @@
+.. SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+..
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+.. _ptbench.results.optimization.signstotb:
+
+==============================
+ SignsToTB model optimization
+==============================
+
+.. note::
+
+ The SignsToTB model contains 161 parameters.
+
+
+SignsToTB is a shallow model created to predict TB presence based on the
+fourteen radiological signs predicted by the DensenetRS model. To train this
+model, we created new features for the Montgomery, Shenzhen and Indian dataset
+by predicting the presence of radiological signs on each of them with
+DensenetRS. Those new datasets versions can be identified by the _RS (for
+Radiological Signs) in their name.
+
+To select the optimal learning rate and the optimal number of neurons for the
+SignsToTB model, we did a grid search with the following parameters.
+
+- 2, 5, 10 and 14 neurons
+- learning rate of 1e-2, 1e-3, 1e-4 and 1e-5
+- batch size of 4
+- 1'000 epochs
+
+We systematically used the training set of the combined dataset MC-CH-IN for
+this optimization.
+
+**The minimum validation loss we found is 0.307 by using a learning rate of
+1e-2 and 10 neurons.**
+
+
+Minimum validation loss grid search
+-----------------------------------
+
+.. list-table::
+
+ * - Learning rate
+ - 2 neurons
+ - 5 neurons
+ - 10 neurons
+ - 14 neurons
+ * - 1e-2
+ - 0.310
+ - 0.314
+ - **0.307**
+ - 0.317
+ * - 1e-3
+ - 0.336
+ - 0.315
+ - 0.314
+ - 0.317
+ * - 1e-4
+ - 0.341
+ - 0.309
+ - 0.321
+ - 0.313
+ * - 1e-5
+ - 0.326
+ - 0.357
+ - 0.337
+ - 0.323
+
+
+Other hyperparameters
+=====================
+
+The default Adam optimizer parameters were used: beta_1=0.9, beta_2=0.999,
+epsilon = 1e-8.
diff --git a/doc/results/runtime.rst b/doc/results/runtime.rst
new file mode 100644
index 0000000000000000000000000000000000000000..7f079c959bfe068ea5f6e896f5876fa04fd46ff0
--- /dev/null
+++ b/doc/results/runtime.rst
@@ -0,0 +1,50 @@
+.. SPDX-FileCopyrightText: Copyright © 2023 Idiap Research Institute <contact@idiap.ch>
+..
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+.. _ptbench.results.runtime:
+
+==============================================
+ Models training runtime and memory footprint
+==============================================
+
+The Pasa and the Densenet models were trained on a machine equipped with an 11
+GB GeForce GTX 1080 Ti GPU, an 8-core processor, 48 GB of RAM and Debian 10.
+The Logistic Regression model was trained on a Macbook Pro with an 8-core
+processor, 32 GB of RAM and macOS Big Sur.
+
+Pasa
+----
+
+- Training on MC: 2'000 epochs in 2.5 hours, ~2 GB of CPU memory, ~0.75 GB of
+ GPU memory
+- Training on MC-CH: 2'000 epochs in 17 hours, ~2 GB of CPU memory, ~0.75 GB of
+ GPU memory
+- Training on MC-CH-IN: 2'000 epochs in 16.5 hours, ~2 GB of CPU memory, ~0.75
+ GB of GPU memory
+
+Densenet pretraining
+--------------------
+
+- Training on NIH CXR14: 10 epochs in 12 hours, ~7.2 GB of CPU memory, ~6.4 GB
+ of GPU memory
+
+Densenet fine-tuning
+--------------------
+
+- Training on MC: 300 epochs in 0.5 hours, ~2 GB of CPU memory, ~6.4 GB of GPU
+ memory
+- Training on MC-CH: 300 epochs in 2.5 hours, ~2 GB of CPU memory, ~6.4 GB of
+ GPU memory
+- Training on MC-CH-IN: 300 epochs in 3.5 hours, ~2 GB of CPU memory, ~6.4 GB
+ of GPU memory
+
+Logistic Regression
+-------------------
+
+- Training on MC: 100 epochs in a few seconds, ~17 GB of CPU memory
+- Training on MC-CH: 100 epochs in a few seconds, ~17 GB of CPU memory
+- Training on MC-CH-IN: 100 epochs in a few seconds, ~17 GB of CPU memory
+
+
+.. include:: ../links.rst
diff --git a/doc/img/direct_vs_indirect.png b/doc/usage/img/direct_vs_indirect.png
similarity index 100%
rename from doc/img/direct_vs_indirect.png
rename to doc/usage/img/direct_vs_indirect.png
diff --git a/doc/usage.rst b/doc/usage/index.rst
similarity index 97%
rename from doc/usage.rst
rename to doc/usage/index.rst
index 7dfba5213b48adf2b62fede52de5673a4900d9de..25f614021950baa3546f01c2a3218d966d68557f 100644
--- a/doc/usage.rst
+++ b/doc/usage/index.rst
@@ -82,10 +82,10 @@ Commands
.. toctree::
:maxdepth: 2
- usage/training
- usage/evaluation
- usage/predtojson
- usage/aggregpred
+ training
+ evaluation
+ predtojson
+ aggregpred
.. include:: links.rst