From 0d6091fc0cea6c95e0a5be5e52c6916ac96077ff Mon Sep 17 00:00:00 2001 From: trungkienbkhn Date: Thu, 11 Sep 2025 09:39:02 +0000 Subject: [PATCH] update ressult --- .../anls_pipeline/human-labels.csv | 82 +++++ .../anls_pipeline/per_image_anls.csv | 40 +-- .../cni/IQA_results/brisque_per_image.csv | 57 ---- .../clipiqa+_vitL14_512_per_image.csv | 57 ---- data/task/cni/IQA_results/dbcnn_per_image.csv | 57 ---- data/task/cni/IQA_results/deqa_per_image.csv | 57 ---- .../cni/IQA_results/hyperiqa_per_image.csv | 57 ---- .../task/cni/IQA_results/maniqa_per_image.csv | 57 ---- data/task/cni/IQA_results/musiq_per_image.csv | 57 ---- data/task/cni/IQA_results/nima_per_image.csv | 57 ---- data/task/cni/IQA_results/niqe_per_image.csv | 57 ---- data/task/cni/IQA_results/nrqm_per_image.csv | 57 ---- .../cni/IQA_results/paq2piq_per_image.csv | 57 ---- data/task/cni/IQA_results/piqe_per_image.csv | 57 ---- .../cni/IQA_results/topiq_nr_per_image.csv | 57 ---- .../task/cni/IQA_results/unique_per_image.csv | 57 ---- .../cni/IQA_results/uranker_per_image.csv | 57 ---- data/task/cni/inference_out/README.md | 161 ++++++++++ data/task/cni/inference_out/SUMMARY.md | 37 +++ .../combined_anls/combined_per_image_anls.csv | 241 ++++++++++++++ .../combined_anls/pipeline/pipeline_means.csv | 6 + .../cni/inference_out/crop/field_metrics.csv | 16 + .../cni/inference_out/crop/per_image_anls.csv | 49 +++ .../crop_shadow_cache/field_metrics.csv | 16 + .../crop_shadow_cache/per_image_anls.csv | 49 +++ .../crop_shadow_paddle/field_metrics.csv | 16 + .../crop_shadow_paddle/per_image_anls.csv | 49 +++ .../field_metrics.csv | 16 + .../per_image_anls.csv | 49 +++ .../no_preprocessing/field_metrics.csv | 16 + .../no_preprocessing/per_image_anls.csv | 49 +++ scripts/compute_hallucination_from_anls.py | 303 ++++++++++++++++++ scripts/pipeline_compare.py | 254 +++++++++++++++ 33 files changed, 1429 insertions(+), 875 deletions(-) create mode 100644 data/task/cni/Hallucination/anls_pipeline/human-labels.csv delete mode 100644 data/task/cni/IQA_results/brisque_per_image.csv delete mode 100644 data/task/cni/IQA_results/clipiqa+_vitL14_512_per_image.csv delete mode 100644 data/task/cni/IQA_results/dbcnn_per_image.csv delete mode 100644 data/task/cni/IQA_results/deqa_per_image.csv delete mode 100644 data/task/cni/IQA_results/hyperiqa_per_image.csv delete mode 100644 data/task/cni/IQA_results/maniqa_per_image.csv delete mode 100644 data/task/cni/IQA_results/musiq_per_image.csv delete mode 100644 data/task/cni/IQA_results/nima_per_image.csv delete mode 100644 data/task/cni/IQA_results/niqe_per_image.csv delete mode 100644 data/task/cni/IQA_results/nrqm_per_image.csv delete mode 100644 data/task/cni/IQA_results/paq2piq_per_image.csv delete mode 100644 data/task/cni/IQA_results/piqe_per_image.csv delete mode 100644 data/task/cni/IQA_results/topiq_nr_per_image.csv delete mode 100644 data/task/cni/IQA_results/unique_per_image.csv delete mode 100644 data/task/cni/IQA_results/uranker_per_image.csv create mode 100644 data/task/cni/inference_out/README.md create mode 100644 data/task/cni/inference_out/SUMMARY.md create mode 100644 data/task/cni/inference_out/combined_anls/combined_per_image_anls.csv create mode 100644 data/task/cni/inference_out/combined_anls/pipeline/pipeline_means.csv create mode 100644 data/task/cni/inference_out/crop/field_metrics.csv create mode 100644 data/task/cni/inference_out/crop/per_image_anls.csv create mode 100644 data/task/cni/inference_out/crop_shadow_cache/field_metrics.csv create mode 100644 data/task/cni/inference_out/crop_shadow_cache/per_image_anls.csv create mode 100644 data/task/cni/inference_out/crop_shadow_paddle/field_metrics.csv create mode 100644 data/task/cni/inference_out/crop_shadow_paddle/per_image_anls.csv create mode 100644 data/task/cni/inference_out/crop_shadow_paddle_cache/field_metrics.csv create mode 100644 data/task/cni/inference_out/crop_shadow_paddle_cache/per_image_anls.csv create mode 100644 data/task/cni/inference_out/no_preprocessing/field_metrics.csv create mode 100644 data/task/cni/inference_out/no_preprocessing/per_image_anls.csv create mode 100644 scripts/compute_hallucination_from_anls.py create mode 100644 scripts/pipeline_compare.py diff --git a/data/task/cni/Hallucination/anls_pipeline/human-labels.csv b/data/task/cni/Hallucination/anls_pipeline/human-labels.csv new file mode 100644 index 0000000..ba34cee --- /dev/null +++ b/data/task/cni/Hallucination/anls_pipeline/human-labels.csv @@ -0,0 +1,82 @@ +filename,label +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed_0.jpg,Low +ITE55340318_ID_kat__1__Completed_0.jpg,Low +ITE55340822_passeport_pascal_2028_Completed_0.jpg,High +ITE55341271_C_I_face_Completed_0.jpg,High +ITE55341677_2023_-_CNI_Recto-Verso_Hassina_0.jpg,High +ITE55343297_20250729_172626_Completed_0.jpg,High +ITE55343716_CNI_Completed_0.jpg,High +ITE55343716_CNI_Completed_1.jpg,High +ITE55346966_17537774579547958575370370624241_Completed_0.jpg,Low +ITE55347865_Snapchat-715567440_Completed_0.jpg,Low +ITE55347866_Snapchat-1551171803_Completed_0.jpg,Low +ITE55347926_DHONDT_CNI_0.jpg,High +ITE55348016_image_Completed_0.jpg,High +ITE55348464_IMG-20250721-WA0000_Completed_0.jpg,High +ITE55348878_CNI_RECTO_0.jpg,Low +ITE55348904_cni_verso_0.jpg,High +ITE55349793_2_em_CNI_CHASTAING__0.jpg,High +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS_0.jpg,High +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS_0.jpg,Low +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_0.jpg,Low +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_1.jpg,Low +ITE55351156_Passeport_Completed_0.jpg,High +ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed_0.jpg,High +ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed_1.jpg,High +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS_0.jpg,Low +ITE55352984_CNI_Completed_0.jpg,Low +ITE55352984_CNI_Completed_1.jpg,Low +ITE55354550_20250728_113359_Completed_0.jpg,High +ITE55354891_image_Completed_0.jpg,High +image (1)_0.jpg,High +image (1)_1.jpg,High +image (10)_0.jpg,Low +image (10)_1.jpg,Low +image (11)_0.jpg,Low +image (11)_1.jpg,Low +image (12)_0.jpg,High +image (12)_1.jpg,High +image (13)_0.jpg,High +image (13)_1.jpg,High +image (14)_0.jpg,High +image (15)_0.jpg,High +image (16)_0.jpg,High +image (17)_0.jpg,High +image (17)_1.jpg,High +image (18)_0.jpg,High +image (18)_1.jpg,High +image (18)_2.jpg,High +image (18)_3.jpg,Low +image (19)_0.jpg,High +image (19)_1.jpg,High +image (2)_0.jpg,High +image (21)_0.jpg,High +image (22)_0.jpg,High +image (22)_1.jpg,High +image (22)_2.jpg,High +image (22)_3.jpg,High +image (23)_0.jpg,High +image (24)_0.jpg,High +image (25)_0.jpg,High +image (26)_0.jpg,High +image (27)_0.jpg,High +image (28)_0.jpg,High +image (28)_1.jpg,High +image (29)_0.jpg,High +image (3)_0.jpg,High +image (30)_0.jpg,High +image (31)_0.jpg,High +image (4)_0.jpg,High +image (5)_0.jpg,Low +image (6)_0.jpg,High +image (6)_1.jpg,High +image (6)_2.jpg,High +image (6)_3.jpg,Low +image (6)_4.jpg,High +image (6)_5.jpg,High +image (6)_6.jpg,High +image (7)_0.jpg,High +image (7)_1.jpg,High +image (8)_0.jpg,High +image (9)_0.jpg,High +image_0.jpg,High diff --git a/data/task/cni/Hallucination/anls_pipeline/per_image_anls.csv b/data/task/cni/Hallucination/anls_pipeline/per_image_anls.csv index 4be8da3..b7f4473 100644 --- a/data/task/cni/Hallucination/anls_pipeline/per_image_anls.csv +++ b/data/task/cni/Hallucination/anls_pipeline/per_image_anls.csv @@ -1,24 +1,4 @@ image,anls,hallucination_score,num_fields -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,0.414286,0.585714,14 -ITE55340318_ID_kat__1__Completed,0.851677,0.148323,15 -ITE55340822_passeport_pascal_2028_Completed,0.857143,0.142857,14 -ITE55341271_C_I_face_Completed,0.888889,0.111111,18 -ITE55346966_17537774579547958575370370624241_Completed,0.400000,0.600000,5 -ITE55347865_Snapchat-715567440_Completed,0.803922,0.196078,17 -ITE55347866_Snapchat-1551171803_Completed,0.774928,0.225072,18 -ITE55347926_DHONDT_CNI,0.931624,0.068376,18 -ITE55348016_image_Completed,0.882353,0.117647,17 -ITE55348464_IMG-20250721-WA0000_Completed,0.583492,0.416508,15 -ITE55348878_CNI_RECTO,0.991900,0.008100,17 -ITE55349793_2_em_CNI_CHASTAING_,0.760000,0.240000,15 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,0.723529,0.276471,17 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,0.848366,0.151634,17 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,0.812707,0.187293,14 -ITE55351156_Passeport_Completed,0.761364,0.238636,16 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,0.428571,0.571429,14 -ITE55352984_CNI_Completed,0.793939,0.206061,15 -ITE55354550_20250728_113359_Completed,0.843712,0.156288,14 -ITE55354891_image_Completed,0.882353,0.117647,17 image (1),0.855357,0.144643,14 image (11),0.440000,0.560000,15 image (12),0.834921,0.165079,14 @@ -46,3 +26,23 @@ image (6),0.853383,0.146617,14 image (8),0.843367,0.156633,14 image (9),0.661905,0.338095,15 image,0.367946,0.632054,15 +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,0.414286,0.585714,14 +ITE55340318_ID_kat__1__Completed,0.851677,0.148323,15 +ITE55340822_passeport_pascal_2028_Completed,0.857143,0.142857,14 +ITE55341271_C_I_face_Completed,0.888889,0.111111,18 +ITE55346966_17537774579547958575370370624241_Completed,0.400000,0.600000,5 +ITE55347865_Snapchat-715567440_Completed,0.803922,0.196078,17 +ITE55347866_Snapchat-1551171803_Completed,0.774928,0.225072,18 +ITE55347926_DHONDT_CNI,0.931624,0.068376,18 +ITE55348016_image_Completed,0.882353,0.117647,17 +ITE55348464_IMG-20250721-WA0000_Completed,0.583492,0.416508,15 +ITE55348878_CNI_RECTO,0.991900,0.008100,17 +ITE55349793_2_em_CNI_CHASTAING_,0.760000,0.240000,15 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,0.723529,0.276471,17 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,0.848366,0.151634,17 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,0.812707,0.187293,14 +ITE55351156_Passeport_Completed,0.761364,0.238636,16 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,0.428571,0.571429,14 +ITE55352984_CNI_Completed,0.793939,0.206061,15 +ITE55354550_20250728_113359_Completed,0.843712,0.156288,14 +ITE55354891_image_Completed,0.882353,0.117647,17 \ No newline at end of file diff --git a/data/task/cni/IQA_results/brisque_per_image.csv b/data/task/cni/IQA_results/brisque_per_image.csv deleted file mode 100644 index dcecaf5..0000000 --- a/data/task/cni/IQA_results/brisque_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,33.205000 -ITE55340318_ID_kat__1__Completed,52.446000 -ITE55340822_passeport_pascal_2028_Completed,67.004000 -ITE55341271_C_I_face_Completed,81.573000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,35.273000 -ITE55343297_20250729_172626_Completed,65.533000 -ITE55343716_CNI_Completed,101.605000 -ITE55346966_17537774579547958575370370624241_Completed,58.592000 -ITE55347865_Snapchat-715567440_Completed,60.966000 -ITE55347866_Snapchat-1551171803_Completed,62.116000 -ITE55347926_DHONDT_CNI,115.342000 -ITE55348016_image_Completed,47.489000 -ITE55348464_IMG-20250721-WA0000_Completed,95.038000 -ITE55348878_CNI_RECTO,64.553000 -ITE55348904_cni_verso,59.017000 -ITE55349793_2_em_CNI_CHASTAING_,76.403000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,166.064000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,109.912000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,118.884500 -ITE55351156_Passeport_Completed,48.598000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,66.095500 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,171.898000 -ITE55352984_CNI_Completed,21.155000 -ITE55354550_20250728_113359_Completed,58.540000 -ITE55354891_image_Completed,54.287000 -image (1),74.796000 -image (10),88.621000 -image (11),18.107500 -image (12),74.987000 -image (13),76.744000 -image (14),83.378000 -image (15),45.582000 -image (16),85.026000 -image (17),88.222000 -image (18),88.733250 -image (19),88.646500 -image (2),69.199000 -image (21),56.357000 -image (22),106.915750 -image (23),40.826000 -image (24),34.812000 -image (25),57.497000 -image (26),110.654000 -image (27),54.174000 -image (28),114.686500 -image (29),60.870000 -image (3),59.187000 -image (30),80.794000 -image (31),85.562000 -image (4),31.299000 -image (5),86.455000 -image (6),69.282714 -image (7),64.166000 -image (8),72.578000 -image (9),47.928000 -image_0,104.085000 diff --git a/data/task/cni/IQA_results/clipiqa+_vitL14_512_per_image.csv b/data/task/cni/IQA_results/clipiqa+_vitL14_512_per_image.csv deleted file mode 100644 index 53a3ab9..0000000 --- a/data/task/cni/IQA_results/clipiqa+_vitL14_512_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,0.256000 -ITE55340318_ID_kat__1__Completed,0.321000 -ITE55340822_passeport_pascal_2028_Completed,0.243000 -ITE55341271_C_I_face_Completed,0.307000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,0.202000 -ITE55343297_20250729_172626_Completed,0.275000 -ITE55343716_CNI_Completed,0.225500 -ITE55346966_17537774579547958575370370624241_Completed,0.241000 -ITE55347865_Snapchat-715567440_Completed,0.180000 -ITE55347866_Snapchat-1551171803_Completed,0.173000 -ITE55347926_DHONDT_CNI,0.515000 -ITE55348016_image_Completed,0.302000 -ITE55348464_IMG-20250721-WA0000_Completed,0.163000 -ITE55348878_CNI_RECTO,0.124000 -ITE55348904_cni_verso,0.132000 -ITE55349793_2_em_CNI_CHASTAING_,0.119000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,0.619000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,0.584000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,0.559500 -ITE55351156_Passeport_Completed,0.256000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,0.295500 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,0.603000 -ITE55352984_CNI_Completed,0.409000 -ITE55354550_20250728_113359_Completed,0.273000 -ITE55354891_image_Completed,0.446000 -image (1),0.178000 -image (10),0.355500 -image (11),0.406000 -image (12),0.337500 -image (13),0.234500 -image (14),0.306000 -image (15),0.466000 -image (16),0.310000 -image (17),0.336000 -image (18),0.375500 -image (19),0.362000 -image (2),0.395000 -image (21),0.367000 -image (22),0.380000 -image (23),0.370000 -image (24),0.361000 -image (25),0.304000 -image (26),0.421000 -image (27),0.266000 -image (28),0.365000 -image (29),0.178000 -image (3),0.252000 -image (30),0.159000 -image (31),0.206000 -image (4),0.123000 -image (5),0.198000 -image (6),0.342143 -image (7),0.289500 -image (8),0.212000 -image (9),0.366000 -image_0,0.432000 diff --git a/data/task/cni/IQA_results/dbcnn_per_image.csv b/data/task/cni/IQA_results/dbcnn_per_image.csv deleted file mode 100644 index 5f6e1a0..0000000 --- a/data/task/cni/IQA_results/dbcnn_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,0.400000 -ITE55340318_ID_kat__1__Completed,0.544000 -ITE55340822_passeport_pascal_2028_Completed,0.420000 -ITE55341271_C_I_face_Completed,0.596000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,0.497000 -ITE55343297_20250729_172626_Completed,0.424000 -ITE55343716_CNI_Completed,0.549000 -ITE55346966_17537774579547958575370370624241_Completed,0.345000 -ITE55347865_Snapchat-715567440_Completed,0.326000 -ITE55347866_Snapchat-1551171803_Completed,0.319000 -ITE55347926_DHONDT_CNI,0.621000 -ITE55348016_image_Completed,0.357000 -ITE55348464_IMG-20250721-WA0000_Completed,0.366000 -ITE55348878_CNI_RECTO,0.327000 -ITE55348904_cni_verso,0.333000 -ITE55349793_2_em_CNI_CHASTAING_,0.285000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,0.571000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,0.548000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,0.556000 -ITE55351156_Passeport_Completed,0.448000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,0.437000 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,0.537000 -ITE55352984_CNI_Completed,0.572500 -ITE55354550_20250728_113359_Completed,0.375000 -ITE55354891_image_Completed,0.370000 -image (1),0.487000 -image (10),0.556000 -image (11),0.498500 -image (12),0.578500 -image (13),0.422000 -image (14),0.489000 -image (15),0.542000 -image (16),0.500000 -image (17),0.580000 -image (18),0.583250 -image (19),0.561500 -image (2),0.398000 -image (21),0.543000 -image (22),0.577500 -image (23),0.435000 -image (24),0.451000 -image (25),0.525000 -image (26),0.638000 -image (27),0.377000 -image (28),0.654500 -image (29),0.392000 -image (3),0.349000 -image (30),0.360000 -image (31),0.356000 -image (4),0.369000 -image (5),0.398000 -image (6),0.440429 -image (7),0.401500 -image (8),0.392000 -image (9),0.443000 -image_0,0.642000 diff --git a/data/task/cni/IQA_results/deqa_per_image.csv b/data/task/cni/IQA_results/deqa_per_image.csv deleted file mode 100644 index 3ea5648..0000000 --- a/data/task/cni/IQA_results/deqa_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,3.000000 -ITE55340318_ID_kat__1__Completed,2.900000 -ITE55340822_passeport_pascal_2028_Completed,3.500000 -ITE55341271_C_I_face_Completed,3.300000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,3.200000 -ITE55343297_20250729_172626_Completed,3.600000 -ITE55343716_CNI_Completed,3.200000 -ITE55346966_17537774579547958575370370624241_Completed,2.800000 -ITE55347865_Snapchat-715567440_Completed,2.800000 -ITE55347866_Snapchat-1551171803_Completed,2.800000 -ITE55347926_DHONDT_CNI,3.100000 -ITE55348016_image_Completed,3.600000 -ITE55348464_IMG-20250721-WA0000_Completed,2.900000 -ITE55348878_CNI_RECTO,2.700000 -ITE55348904_cni_verso,3.400000 -ITE55349793_2_em_CNI_CHASTAING_,2.600000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,2.700000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,2.400000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,1.950000 -ITE55351156_Passeport_Completed,3.900000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,3.500000 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,2.200000 -ITE55352984_CNI_Completed,3.200000 -ITE55354550_20250728_113359_Completed,3.700000 -ITE55354891_image_Completed,3.700000 -image (1),2.850000 -image (10),2.900000 -image (11),2.950000 -image (12),3.400000 -image (13),2.750000 -image (14),2.700000 -image (15),3.700000 -image (16),3.100000 -image (17),3.400000 -image (18),3.375000 -image (19),2.850000 -image (2),3.800000 -image (21),3.700000 -image (22),3.125000 -image (23),3.700000 -image (24),3.500000 -image (25),3.800000 -image (26),3.100000 -image (27),3.500000 -image (28),2.950000 -image (29),3.600000 -image (3),3.800000 -image (30),3.300000 -image (31),2.700000 -image (4),2.900000 -image (5),2.800000 -image (6),3.371429 -image (7),3.550000 -image (8),2.700000 -image (9),3.900000 -image_0,3.000000 diff --git a/data/task/cni/IQA_results/hyperiqa_per_image.csv b/data/task/cni/IQA_results/hyperiqa_per_image.csv deleted file mode 100644 index a539018..0000000 --- a/data/task/cni/IQA_results/hyperiqa_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,0.385000 -ITE55340318_ID_kat__1__Completed,0.500000 -ITE55340822_passeport_pascal_2028_Completed,0.396000 -ITE55341271_C_I_face_Completed,0.458000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,0.383000 -ITE55343297_20250729_172626_Completed,0.337000 -ITE55343716_CNI_Completed,0.573000 -ITE55346966_17537774579547958575370370624241_Completed,0.291000 -ITE55347865_Snapchat-715567440_Completed,0.246000 -ITE55347866_Snapchat-1551171803_Completed,0.296000 -ITE55347926_DHONDT_CNI,0.609000 -ITE55348016_image_Completed,0.282000 -ITE55348464_IMG-20250721-WA0000_Completed,0.489000 -ITE55348878_CNI_RECTO,0.363000 -ITE55348904_cni_verso,0.336000 -ITE55349793_2_em_CNI_CHASTAING_,0.434000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,0.548000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,0.596000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,0.562000 -ITE55351156_Passeport_Completed,0.334000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,0.535000 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,0.557000 -ITE55352984_CNI_Completed,0.452000 -ITE55354550_20250728_113359_Completed,0.311000 -ITE55354891_image_Completed,0.240000 -image (1),0.413000 -image (10),0.561500 -image (11),0.342500 -image (12),0.605500 -image (13),0.504000 -image (14),0.576000 -image (15),0.539000 -image (16),0.597000 -image (17),0.578000 -image (18),0.610750 -image (19),0.551500 -image (2),0.593000 -image (21),0.497000 -image (22),0.598000 -image (23),0.374000 -image (24),0.388000 -image (25),0.522000 -image (26),0.597000 -image (27),0.289000 -image (28),0.607000 -image (29),0.401000 -image (3),0.293000 -image (30),0.426000 -image (31),0.440000 -image (4),0.324000 -image (5),0.483000 -image (6),0.497714 -image (7),0.397000 -image (8),0.482000 -image (9),0.367000 -image_0,0.597000 diff --git a/data/task/cni/IQA_results/maniqa_per_image.csv b/data/task/cni/IQA_results/maniqa_per_image.csv deleted file mode 100644 index 06a763f..0000000 --- a/data/task/cni/IQA_results/maniqa_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,0.252000 -ITE55340318_ID_kat__1__Completed,0.445000 -ITE55340822_passeport_pascal_2028_Completed,0.370000 -ITE55341271_C_I_face_Completed,0.430000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,0.303000 -ITE55343297_20250729_172626_Completed,0.384000 -ITE55343716_CNI_Completed,0.405000 -ITE55346966_17537774579547958575370370624241_Completed,0.362000 -ITE55347865_Snapchat-715567440_Completed,0.197000 -ITE55347866_Snapchat-1551171803_Completed,0.237000 -ITE55347926_DHONDT_CNI,0.426000 -ITE55348016_image_Completed,0.339000 -ITE55348464_IMG-20250721-WA0000_Completed,0.372000 -ITE55348878_CNI_RECTO,0.295000 -ITE55348904_cni_verso,0.303000 -ITE55349793_2_em_CNI_CHASTAING_,0.273000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,0.536000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,0.463000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,0.467500 -ITE55351156_Passeport_Completed,0.381000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,0.329500 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,0.481000 -ITE55352984_CNI_Completed,0.478500 -ITE55354550_20250728_113359_Completed,0.284000 -ITE55354891_image_Completed,0.227000 -image (1),0.343500 -image (10),0.415500 -image (11),0.389500 -image (12),0.446500 -image (13),0.366000 -image (14),0.423000 -image (15),0.501000 -image (16),0.420000 -image (17),0.393500 -image (18),0.460500 -image (19),0.428500 -image (2),0.442000 -image (21),0.349000 -image (22),0.437250 -image (23),0.333000 -image (24),0.376000 -image (25),0.386000 -image (26),0.449000 -image (27),0.309000 -image (28),0.464000 -image (29),0.389000 -image (3),0.347000 -image (30),0.359000 -image (31),0.372000 -image (4),0.328000 -image (5),0.363000 -image (6),0.397714 -image (7),0.375000 -image (8),0.356000 -image (9),0.340000 -image_0,0.457000 diff --git a/data/task/cni/IQA_results/musiq_per_image.csv b/data/task/cni/IQA_results/musiq_per_image.csv deleted file mode 100644 index 50d7346..0000000 --- a/data/task/cni/IQA_results/musiq_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,35.991000 -ITE55340318_ID_kat__1__Completed,37.754000 -ITE55340822_passeport_pascal_2028_Completed,28.985000 -ITE55341271_C_I_face_Completed,56.156000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,38.107000 -ITE55343297_20250729_172626_Completed,44.047000 -ITE55343716_CNI_Completed,45.377500 -ITE55346966_17537774579547958575370370624241_Completed,20.196000 -ITE55347865_Snapchat-715567440_Completed,23.950000 -ITE55347866_Snapchat-1551171803_Completed,22.812000 -ITE55347926_DHONDT_CNI,49.142000 -ITE55348016_image_Completed,29.268000 -ITE55348464_IMG-20250721-WA0000_Completed,27.107000 -ITE55348878_CNI_RECTO,18.812000 -ITE55348904_cni_verso,29.798000 -ITE55349793_2_em_CNI_CHASTAING_,22.087000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,41.226000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,39.767000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,42.864500 -ITE55351156_Passeport_Completed,36.533000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,39.081500 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,38.843000 -ITE55352984_CNI_Completed,50.209500 -ITE55354550_20250728_113359_Completed,24.420000 -ITE55354891_image_Completed,24.471000 -image (1),46.430000 -image (10),50.053500 -image (11),46.356500 -image (12),52.554000 -image (13),42.773500 -image (14),47.782000 -image (15),62.877000 -image (16),61.844000 -image (17),58.803000 -image (18),51.741500 -image (19),55.454000 -image (2),42.548000 -image (21),52.730000 -image (22),49.552500 -image (23),52.865000 -image (24),53.499000 -image (25),56.627000 -image (26),58.491000 -image (27),36.273000 -image (28),56.751500 -image (29),38.741000 -image (3),35.724000 -image (30),32.215000 -image (31),28.717000 -image (4),22.798000 -image (5),29.930000 -image (6),45.327571 -image (7),34.085000 -image (8),46.315000 -image (9),44.764000 -image_0,56.602000 diff --git a/data/task/cni/IQA_results/nima_per_image.csv b/data/task/cni/IQA_results/nima_per_image.csv deleted file mode 100644 index 51ef775..0000000 --- a/data/task/cni/IQA_results/nima_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,3.695000 -ITE55340318_ID_kat__1__Completed,3.929000 -ITE55340822_passeport_pascal_2028_Completed,4.098000 -ITE55341271_C_I_face_Completed,4.517000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,4.311000 -ITE55343297_20250729_172626_Completed,4.993000 -ITE55343716_CNI_Completed,4.241000 -ITE55346966_17537774579547958575370370624241_Completed,3.807000 -ITE55347865_Snapchat-715567440_Completed,4.464000 -ITE55347866_Snapchat-1551171803_Completed,4.632000 -ITE55347926_DHONDT_CNI,3.641000 -ITE55348016_image_Completed,4.653000 -ITE55348464_IMG-20250721-WA0000_Completed,4.113000 -ITE55348878_CNI_RECTO,4.240000 -ITE55348904_cni_verso,4.589000 -ITE55349793_2_em_CNI_CHASTAING_,3.340000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,3.585000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,3.833000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,3.719000 -ITE55351156_Passeport_Completed,4.196000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,4.101000 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,3.678000 -ITE55352984_CNI_Completed,4.288500 -ITE55354550_20250728_113359_Completed,4.644000 -ITE55354891_image_Completed,5.304000 -image (1),3.905500 -image (10),4.137000 -image (11),3.828500 -image (12),4.297000 -image (13),4.226000 -image (14),4.511000 -image (15),4.962000 -image (16),4.005000 -image (17),3.997000 -image (18),3.637000 -image (19),4.973000 -image (2),4.808000 -image (21),4.258000 -image (22),3.557000 -image (23),4.804000 -image (24),4.095000 -image (25),4.132000 -image (26),4.208000 -image (27),5.041000 -image (28),3.902000 -image (29),4.357000 -image (3),4.368000 -image (30),4.218000 -image (31),4.696000 -image (4),4.096000 -image (5),3.907000 -image (6),4.402286 -image (7),4.632500 -image (8),4.253000 -image (9),4.467000 -image_0,4.479000 diff --git a/data/task/cni/IQA_results/niqe_per_image.csv b/data/task/cni/IQA_results/niqe_per_image.csv deleted file mode 100644 index e8be077..0000000 --- a/data/task/cni/IQA_results/niqe_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,3.322000 -ITE55340318_ID_kat__1__Completed,6.695000 -ITE55340822_passeport_pascal_2028_Completed,8.592000 -ITE55341271_C_I_face_Completed,6.091000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,4.221000 -ITE55343297_20250729_172626_Completed,8.058000 -ITE55343716_CNI_Completed,6.535000 -ITE55346966_17537774579547958575370370624241_Completed,7.704000 -ITE55347865_Snapchat-715567440_Completed,8.664000 -ITE55347866_Snapchat-1551171803_Completed,8.490000 -ITE55347926_DHONDT_CNI,8.006000 -ITE55348016_image_Completed,7.674000 -ITE55348464_IMG-20250721-WA0000_Completed,8.749000 -ITE55348878_CNI_RECTO,10.474000 -ITE55348904_cni_verso,9.687000 -ITE55349793_2_em_CNI_CHASTAING_,5.550000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,16.735000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,20.799000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,19.900000 -ITE55351156_Passeport_Completed,7.712000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,9.527500 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,16.535000 -ITE55352984_CNI_Completed,4.465000 -ITE55354550_20250728_113359_Completed,7.836000 -ITE55354891_image_Completed,8.538000 -image (1),6.078500 -image (10),5.823500 -image (11),4.638500 -image (12),7.236500 -image (13),6.585500 -image (14),8.515000 -image (15),4.175000 -image (16),7.439000 -image (17),6.496000 -image (18),7.827500 -image (19),6.848000 -image (2),13.407000 -image (21),4.337000 -image (22),11.914250 -image (23),4.087000 -image (24),4.123000 -image (25),5.168000 -image (26),6.608000 -image (27),8.346000 -image (28),10.653500 -image (29),6.459000 -image (3),8.250000 -image (30),9.326000 -image (31),8.102000 -image (4),5.555000 -image (5),10.295000 -image (6),12.200143 -image (7),8.800000 -image (8),6.882000 -image (9),4.001000 -image_0,6.679000 diff --git a/data/task/cni/IQA_results/nrqm_per_image.csv b/data/task/cni/IQA_results/nrqm_per_image.csv deleted file mode 100644 index 9c00fd1..0000000 --- a/data/task/cni/IQA_results/nrqm_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,6.774000 -ITE55340318_ID_kat__1__Completed,5.434000 -ITE55340822_passeport_pascal_2028_Completed,4.183000 -ITE55341271_C_I_face_Completed,5.837000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,6.264000 -ITE55343297_20250729_172626_Completed,4.258000 -ITE55343716_CNI_Completed,6.657500 -ITE55346966_17537774579547958575370370624241_Completed,4.061000 -ITE55347865_Snapchat-715567440_Completed,3.856000 -ITE55347866_Snapchat-1551171803_Completed,4.436000 -ITE55347926_DHONDT_CNI,6.428000 -ITE55348016_image_Completed,4.995000 -ITE55348464_IMG-20250721-WA0000_Completed,5.353000 -ITE55348878_CNI_RECTO,3.692000 -ITE55348904_cni_verso,3.795000 -ITE55349793_2_em_CNI_CHASTAING_,5.226000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,6.031000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,6.434000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,6.197000 -ITE55351156_Passeport_Completed,5.300000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,6.011500 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,6.633000 -ITE55352984_CNI_Completed,6.938000 -ITE55354550_20250728_113359_Completed,3.901000 -ITE55354891_image_Completed,3.896000 -image (1),5.805000 -image (10),5.604500 -image (11),6.663000 -image (12),6.572500 -image (13),6.217500 -image (14),5.275000 -image (15),6.800000 -image (16),5.230000 -image (17),5.722500 -image (18),6.446750 -image (19),6.351500 -image (2),4.837000 -image (21),5.707000 -image (22),6.130000 -image (23),5.047000 -image (24),5.141000 -image (25),5.661000 -image (26),6.152000 -image (27),5.087000 -image (28),6.548000 -image (29),4.495000 -image (3),4.310000 -image (30),4.171000 -image (31),3.966000 -image (4),5.986000 -image (5),5.872000 -image (6),5.381571 -image (7),4.758000 -image (8),4.964000 -image (9),5.884000 -image_0,6.518000 diff --git a/data/task/cni/IQA_results/paq2piq_per_image.csv b/data/task/cni/IQA_results/paq2piq_per_image.csv deleted file mode 100644 index 798f6e5..0000000 --- a/data/task/cni/IQA_results/paq2piq_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,64.096000 -ITE55340318_ID_kat__1__Completed,60.963000 -ITE55340822_passeport_pascal_2028_Completed,62.171000 -ITE55341271_C_I_face_Completed,65.879000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,69.519000 -ITE55343297_20250729_172626_Completed,65.233000 -ITE55343716_CNI_Completed,64.196500 -ITE55346966_17537774579547958575370370624241_Completed,54.323000 -ITE55347865_Snapchat-715567440_Completed,46.831000 -ITE55347866_Snapchat-1551171803_Completed,47.426000 -ITE55347926_DHONDT_CNI,64.576000 -ITE55348016_image_Completed,53.730000 -ITE55348464_IMG-20250721-WA0000_Completed,63.127000 -ITE55348878_CNI_RECTO,60.722000 -ITE55348904_cni_verso,60.081000 -ITE55349793_2_em_CNI_CHASTAING_,56.292000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,63.136000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,66.946000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,63.195500 -ITE55351156_Passeport_Completed,69.644000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,61.447000 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,59.149000 -ITE55352984_CNI_Completed,71.036500 -ITE55354550_20250728_113359_Completed,58.336000 -ITE55354891_image_Completed,60.885000 -image (1),63.573000 -image (10),64.958000 -image (11),68.192000 -image (12),68.098000 -image (13),62.931500 -image (14),65.586000 -image (15),67.515000 -image (16),67.091000 -image (17),64.243500 -image (18),62.446250 -image (19),65.734500 -image (2),64.490000 -image (21),65.695000 -image (22),65.092000 -image (23),64.503000 -image (24),70.368000 -image (25),62.184000 -image (26),66.834000 -image (27),62.969000 -image (28),64.736000 -image (29),67.307000 -image (3),65.400000 -image (30),62.833000 -image (31),62.583000 -image (4),64.296000 -image (5),59.851000 -image (6),63.609857 -image (7),58.294500 -image (8),63.383000 -image (9),68.576000 -image_0,67.255000 diff --git a/data/task/cni/IQA_results/piqe_per_image.csv b/data/task/cni/IQA_results/piqe_per_image.csv deleted file mode 100644 index 0dc4d37..0000000 --- a/data/task/cni/IQA_results/piqe_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,45.286000 -ITE55340318_ID_kat__1__Completed,72.576000 -ITE55340822_passeport_pascal_2028_Completed,72.649000 -ITE55341271_C_I_face_Completed,43.073000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,65.407000 -ITE55343297_20250729_172626_Completed,73.591000 -ITE55343716_CNI_Completed,36.356500 -ITE55346966_17537774579547958575370370624241_Completed,84.354000 -ITE55347865_Snapchat-715567440_Completed,84.337000 -ITE55347866_Snapchat-1551171803_Completed,82.886000 -ITE55347926_DHONDT_CNI,32.465000 -ITE55348016_image_Completed,57.182000 -ITE55348464_IMG-20250721-WA0000_Completed,72.738000 -ITE55348878_CNI_RECTO,75.959000 -ITE55348904_cni_verso,74.901000 -ITE55349793_2_em_CNI_CHASTAING_,61.437000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,83.426000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,83.921000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,83.822000 -ITE55351156_Passeport_Completed,70.436000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,29.469000 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,81.790000 -ITE55352984_CNI_Completed,37.070500 -ITE55354550_20250728_113359_Completed,88.094000 -ITE55354891_image_Completed,79.152000 -image (1),40.994000 -image (10),33.116000 -image (11),51.679500 -image (12),40.610500 -image (13),72.858500 -image (14),45.315000 -image (15),58.959000 -image (16),54.716000 -image (17),57.000500 -image (18),54.890750 -image (19),53.274500 -image (2),68.054000 -image (21),56.864000 -image (22),63.000500 -image (23),47.615000 -image (24),49.904000 -image (25),59.749000 -image (26),38.403000 -image (27),79.029000 -image (28),56.642000 -image (29),66.151000 -image (3),81.799000 -image (30),73.928000 -image (31),75.877000 -image (4),38.801000 -image (5),72.868000 -image (6),57.162000 -image (7),80.672000 -image (8),68.791000 -image (9),59.924000 -image_0,40.057000 diff --git a/data/task/cni/IQA_results/topiq_nr_per_image.csv b/data/task/cni/IQA_results/topiq_nr_per_image.csv deleted file mode 100644 index 874e8ab..0000000 --- a/data/task/cni/IQA_results/topiq_nr_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,0.340000 -ITE55340318_ID_kat__1__Completed,0.473000 -ITE55340822_passeport_pascal_2028_Completed,0.333000 -ITE55341271_C_I_face_Completed,0.567000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,0.393000 -ITE55343297_20250729_172626_Completed,0.402000 -ITE55343716_CNI_Completed,0.503500 -ITE55346966_17537774579547958575370370624241_Completed,0.275000 -ITE55347865_Snapchat-715567440_Completed,0.233000 -ITE55347866_Snapchat-1551171803_Completed,0.254000 -ITE55347926_DHONDT_CNI,0.610000 -ITE55348016_image_Completed,0.306000 -ITE55348464_IMG-20250721-WA0000_Completed,0.319000 -ITE55348878_CNI_RECTO,0.263000 -ITE55348904_cni_verso,0.305000 -ITE55349793_2_em_CNI_CHASTAING_,0.221000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,0.471000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,0.461000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,0.399500 -ITE55351156_Passeport_Completed,0.351000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,0.404500 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,0.415000 -ITE55352984_CNI_Completed,0.463000 -ITE55354550_20250728_113359_Completed,0.284000 -ITE55354891_image_Completed,0.290000 -image (1),0.371000 -image (10),0.531500 -image (11),0.351000 -image (12),0.560000 -image (13),0.372000 -image (14),0.456000 -image (15),0.535000 -image (16),0.470000 -image (17),0.533000 -image (18),0.592000 -image (19),0.486500 -image (2),0.454000 -image (21),0.491000 -image (22),0.522000 -image (23),0.378000 -image (24),0.409000 -image (25),0.492000 -image (26),0.619000 -image (27),0.341000 -image (28),0.609500 -image (29),0.347000 -image (3),0.318000 -image (30),0.313000 -image (31),0.344000 -image (4),0.323000 -image (5),0.357000 -image (6),0.434714 -image (7),0.345500 -image (8),0.337000 -image (9),0.399000 -image_0,0.627000 diff --git a/data/task/cni/IQA_results/unique_per_image.csv b/data/task/cni/IQA_results/unique_per_image.csv deleted file mode 100644 index fb6ced7..0000000 --- a/data/task/cni/IQA_results/unique_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,-0.006000 -ITE55340318_ID_kat__1__Completed,0.340000 -ITE55340822_passeport_pascal_2028_Completed,-0.604000 -ITE55341271_C_I_face_Completed,0.799000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,0.325000 -ITE55343297_20250729_172626_Completed,-0.699000 -ITE55343716_CNI_Completed,0.442500 -ITE55346966_17537774579547958575370370624241_Completed,-0.691000 -ITE55347865_Snapchat-715567440_Completed,-1.288000 -ITE55347866_Snapchat-1551171803_Completed,-1.193000 -ITE55347926_DHONDT_CNI,0.562000 -ITE55348016_image_Completed,-0.779000 -ITE55348464_IMG-20250721-WA0000_Completed,-0.693000 -ITE55348878_CNI_RECTO,-0.767000 -ITE55348904_cni_verso,-0.630000 -ITE55349793_2_em_CNI_CHASTAING_,-1.019000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,-0.581000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,-0.226000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,-0.327500 -ITE55351156_Passeport_Completed,-0.116000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,0.407500 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,-0.464000 -ITE55352984_CNI_Completed,0.421000 -ITE55354550_20250728_113359_Completed,-0.662000 -ITE55354891_image_Completed,-0.685000 -image (1),0.289000 -image (10),0.996000 -image (11),0.495000 -image (12),0.931000 -image (13),-0.207000 -image (14),0.650000 -image (15),0.621000 -image (16),0.815000 -image (17),1.049500 -image (18),0.684000 -image (19),0.995000 -image (2),-0.014000 -image (21),0.737000 -image (22),0.562750 -image (23),0.375000 -image (24),0.516000 -image (25),1.031000 -image (26),0.855000 -image (27),-0.701000 -image (28),0.721500 -image (29),-0.014000 -image (3),-0.576000 -image (30),-0.530000 -image (31),-0.805000 -image (4),-0.161000 -image (5),0.045000 -image (6),0.196429 -image (7),-0.422000 -image (8),-0.762000 -image (9),0.376000 -image_0,0.597000 diff --git a/data/task/cni/IQA_results/uranker_per_image.csv b/data/task/cni/IQA_results/uranker_per_image.csv deleted file mode 100644 index 34036db..0000000 --- a/data/task/cni/IQA_results/uranker_per_image.csv +++ /dev/null @@ -1,57 +0,0 @@ -image,score -ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,0.560000 -ITE55340318_ID_kat__1__Completed,0.897000 -ITE55340822_passeport_pascal_2028_Completed,0.205000 -ITE55341271_C_I_face_Completed,-0.494000 -ITE55341677_2023_-_CNI_Recto-Verso_Hassina,0.428000 -ITE55343297_20250729_172626_Completed,2.212000 -ITE55343716_CNI_Completed,-0.442000 -ITE55346966_17537774579547958575370370624241_Completed,-0.420000 -ITE55347865_Snapchat-715567440_Completed,0.133000 -ITE55347866_Snapchat-1551171803_Completed,-0.380000 -ITE55347926_DHONDT_CNI,-0.752000 -ITE55348016_image_Completed,0.308000 -ITE55348464_IMG-20250721-WA0000_Completed,0.526000 -ITE55348878_CNI_RECTO,-0.387000 -ITE55348904_cni_verso,0.412000 -ITE55349793_2_em_CNI_CHASTAING_,1.076000 -ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,1.679000 -ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,0.686000 -ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,-0.049500 -ITE55351156_Passeport_Completed,1.070000 -ITE55351372_RÉPUBLIQUE_FRANÇAISE_Completed,0.619500 -ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,0.503000 -ITE55352984_CNI_Completed,1.951000 -ITE55354550_20250728_113359_Completed,0.673000 -ITE55354891_image_Completed,1.477000 -image (1),-0.750000 -image (10),-0.785000 -image (11),1.869500 -image (12),-0.240000 -image (13),-0.220000 -image (14),-0.982000 -image (15),1.902000 -image (16),-0.625000 -image (17),-0.843000 -image (18),0.215000 -image (19),-0.456000 -image (2),-0.104000 -image (21),-0.082000 -image (22),-0.823750 -image (23),-0.527000 -image (24),-0.621000 -image (25),0.584000 -image (26),-0.665000 -image (27),1.514000 -image (28),-0.559000 -image (29),0.436000 -image (3),1.118000 -image (30),0.032000 -image (31),-0.285000 -image (4),0.963000 -image (5),-0.799000 -image (6),-0.161000 -image (7),0.468500 -image (8),0.265000 -image (9),1.677000 -image_0,-0.574000 diff --git a/data/task/cni/inference_out/README.md b/data/task/cni/inference_out/README.md new file mode 100644 index 0000000..b89bbcc --- /dev/null +++ b/data/task/cni/inference_out/README.md @@ -0,0 +1,161 @@ +# Document Field Extraction Evaluation Results + +## Overview + +This document presents the evaluation results for document field extraction using different preprocessing approaches. The evaluation was conducted on a dataset of 56 document samples with various field types commonly found in identity documents. + +## Evaluation Metrics + +The evaluation uses standard information extraction metrics: +- **Precision**: Ratio of correctly extracted fields to total extracted fields +- **Recall**: Ratio of correctly extracted fields to total ground truth fields +- **F1-Score**: Harmonic mean of precision and recall +- **Accuracy**: Overall field-level accuracy +- **TP**: True Positives (correctly extracted fields) +- **FP**: False Positives (incorrectly extracted fields) +- **FN**: False Negatives (missed fields) + +## Preprocessing Approaches + +### 1. No Preprocessing (Baseline) +- **Configuration**: Raw images without any preprocessing +- **Performance**: + - Micro Precision: 79.0% + - Micro Recall: 68.7% + - Micro F1: 73.5% + - Field Accuracy: 68.7% + +### 2. Crop +- **Configuration**: Content-aware cropping (no shadow removal) +- **Performance**: + - Micro Precision: 94.8% + - Micro Recall: 89.9% + - Micro F1: 92.3% (+18.8% improvement) + - Field Accuracy: 89.9% + +### 3. Crop + PaddleOCR + Shadow Removal +- **Configuration**: Cropping with PaddleOCR document processing and shadow removal +- **Performance**: + - Micro Precision: 93.6% + - Micro Recall: 89.4% + - Micro F1: 91.5% (+18.0% improvement) + - Field Accuracy: 89.4% + +### 4. Crop + PaddleOCR + Shadow Removal + Cache +- **Configuration**: Cropping with PaddleOCR, shadow removal, and caching +- **Performance**: + - Micro Precision: 92.5% + - Micro Recall: 88.3% + - Micro F1: 90.3% (+16.8% improvement) + - Field Accuracy: 88.3% + +### 5. Crop + Shadow Removal + Cache +- **Configuration**: Cropping with shadow removal and caching +- **Performance**: + - Micro Precision: 93.6% + - Micro Recall: 88.5% + - Micro F1: 91.0% (+17.5% improvement) + - Field Accuracy: 88.5% + +## Field-Level Performance Analysis + +### High-Performance Fields +Fields that consistently perform well across all approaches: + +| Field | Best F1 | Best Approach | Performance Trend | +|-------|----------|---------------|-------------------| +| **Gender** | 85.1% | Crop + PaddleOCR | Consistent improvement | +| **Birth Date** | 80.5% | Crop + PaddleOCR | Strong improvement | +| **Document Type** | 85.4% | Crop + PaddleOCR | Significant improvement | +| **Surname** | 82.9% | Crop + PaddleOCR | Consistent improvement | + +### Medium-Performance Fields +Fields with moderate improvement: + +| Field | Best F1 | Best Approach | Performance Trend | +|-------|----------|---------------|-------------------| +| **Birth Place** | 83.4% | Crop Only | Good improvement | +| **Expiry Date** | 78.5% | Crop + PaddleOCR | Moderate improvement | +| **Issue Date** | 69.3% | Crop + Shadow + Cache | Variable performance | +| **Address** | 44.4% | Crop + PaddleOCR | Limited improvement | + +### Low-Performance Fields +Fields that remain challenging: + +| Field | Best F1 | Best Approach | Notes | +|-------|----------|---------------|-------| +| **MRZ Lines** | 41.8% | Crop + Shadow + Cache | Complex OCR patterns | +| **Personal Number** | 40.0% | Crop + PaddleOCR + Cache | Small text, variable format | +| **Issue Place** | 50.0% | Crop + PaddleOCR + Cache | Handwritten text challenges | + +### Zero-Performance Fields +Fields that consistently fail across all approaches: +- **Recto/Verso**: Document side detection +- **Code**: Encoded information +- **Height**: Physical measurements +- **Type**: Document classification + +## Key Findings + +### 1. Preprocessing Impact +- **Cropping alone** delivers the strongest overall boost (+18.8 F1 pts vs. baseline) +- **PaddleOCR + Shadow Removal** is highly competitive (up to +18.0 F1 pts) +- **Caching** has minimal to moderate impact on accuracy + +### 2. Field Type Sensitivity +- **Structured fields** (dates, numbers) benefit most from preprocessing +- **Text fields** (names, addresses) show moderate improvement +- **Complex fields** (MRZ, codes) remain challenging + +### 3. Processing Pipeline Efficiency +- **Crop** currently provides the best overall F1 in this evaluation +- **Crop + PaddleOCR + Shadow Removal** is close and benefits some fields +- **Caching** shows minimal gains; use for speed, not accuracy + +## Recommendations + +### For Production Use +1. **Use Crop** as the primary preprocessing step +2. **Focus optimization** on high-value fields (dates, document types, names) +3. **Consider field-specific** preprocessing strategies for challenging fields + +### For Further Research +1. **Investigate MRZ line** extraction techniques +2. **Explore advanced OCR** methods for handwritten text +3. **Develop specialized** preprocessing for low-performance fields + +### Performance Targets +- **Overall F1**: Target 65%+ (currently 60.7% best) +- **Field Accuracy**: Target 50%+ (currently 43.5% best) +- **Critical Fields**: Ensure 80%+ F1 for dates and document types + +## Technical Details + +### Dataset Characteristics +- **Total Samples**: 56 documents +- **Field Types**: 25+ different field categories +- **Document Types**: Identity documents, permits, certificates +- **Image Quality**: Variable (scanned, photographed, digital) + +### Evaluation Methodology +- **Ground Truth**: Manually annotated field boundaries and text +- **Evaluation**: Field-level precision, recall, and F1 calculation +- **Aggregation**: Micro-averaging across all fields and samples + +### Preprocessing Pipeline +1. **Image Input**: Raw document images +2. **Cropping**: Content area detection and extraction +3. **Document Processing**: PaddleOCR unwarping and orientation +4. **Shadow Removal**: Optional DocShadow processing +5. **Field Extraction**: OCR-based text extraction +6. **Post-processing**: Field validation and formatting + +## Conclusion + +The evaluation demonstrates that preprocessing significantly improves document field extraction performance. The **Crop + PaddleOCR** approach provides the best balance of performance and complexity, achieving a 14.1% improvement in F1-score over the baseline. While some fields remain challenging, the overall pipeline shows strong potential for production deployment with further field-specific optimizations. + +--- + +*Last Updated: August 2024* +*Evaluation Dataset: 56 document samples* +*Total Fields Evaluated: 900+ field instances* diff --git a/data/task/cni/inference_out/SUMMARY.md b/data/task/cni/inference_out/SUMMARY.md new file mode 100644 index 0000000..fbcb23a --- /dev/null +++ b/data/task/cni/inference_out/SUMMARY.md @@ -0,0 +1,37 @@ +# Evaluation Results Summary + +## Quick Overview +- **Dataset**: 56 document samples +- **Best Approach**: Crop (No Shadow Removal) +- **Performance Gain**: +14.1% F1-score improvement over baseline + +## Performance Comparison (Ranked from Lowest to Highest) + +| Approach | Precision | Recall | F1-Score | Field Accuracy | Improvement vs. Baseline | +|----------|-----------|--------|----------|----------------|---------------------------| +| **No Preprocessing** | 79.0% | 68.7% | 73.5% | 68.7% | Baseline | +| **Crop + PaddleOCR + Shadow Removal + Cache** | 92.5% | 88.3% | 90.3% | 88.3% | +16.8% | +| **Crop + Shadow Removal + Cache** | 93.6% | 88.5% | 91.0% | 88.5% | +17.5% | +| **Crop + PaddleOCR + Shadow Removal** | 93.6% | 89.4% | 91.5% | 89.4% | +18.0% | +| **Crop** | 94.8% | 89.9% | 92.3% | 89.9% | +18.8% | + +## Top Performing Fields +- **Gender**: 85.1% F1 (Crop + PaddleOCR + Shadow Removal) +- **Birth Date**: 80.5% F1 (Crop + PaddleOCR + Shadow Removal) +- **Document Type**: 85.4% F1 (Crop + PaddleOCR + Shadow Removal) +- **Surname**: 82.9% F1 (Crop + PaddleOCR + Shadow Removal) + +## Key Insights +1. **Cropping** provides the biggest performance boost +2. **PaddleOCR + Shadow Removal** adds small but consistent improvement +3. **Shadow removal** shows mixed results depending on field type +4. **Caching** has minimal impact on accuracy + +## Recommendations +- Use **Crop + PaddleOCR + Shadow Removal** for production +- Focus on optimizing high-value fields +- Investigate MRZ line extraction further +- Target 65%+ overall F1-score + +--- +*See README.md for detailed analysis* diff --git a/data/task/cni/inference_out/combined_anls/combined_per_image_anls.csv b/data/task/cni/inference_out/combined_anls/combined_per_image_anls.csv new file mode 100644 index 0000000..6e27b2e --- /dev/null +++ b/data/task/cni/inference_out/combined_anls/combined_per_image_anls.csv @@ -0,0 +1,241 @@ +image,method,anls,hallucination_score,num_fields +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed_,crop,1.0,0.0,12 +ITE55340318_ID_kat__1__Completed_,crop,0.9200562169312168,0.0799437830687831,12 +ITE55340822_passeport_pascal_2028_Completed_,crop,1.0,0.0,12 +ITE55341271_C_I_face_Completed_,crop,0.802415458937198,0.197584541062802,10 +ITE55346966_17537774579547958575370370624241_Completed_,crop,0.8177777777777777,0.1822222222222222,5 +ITE55347865_Snapchat-715567440_Completed_,crop,0.9861111111111112,0.0138888888888888,8 +ITE55347866_Snapchat-1551171803_Completed_,crop,0.938471760797342,0.0615282392026579,5 +ITE55347926_DHONDT_CNI_,crop,0.9012345679012346,0.0987654320987654,9 +ITE55348016_image_Completed_,crop,0.975,0.025,8 +ITE55348464_IMG-20250721-WA0000_Completed_,crop,0.833044733044733,0.1669552669552669,11 +ITE55348878_CNI_RECTO_,crop,0.9637408088235294,0.0362591911764705,8 +ITE55349793_2_em_CNI_CHASTAING__,crop,0.988095238095238,0.0119047619047618,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS_,crop,0.9857142857142858,0.0142857142857142,7 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS_,crop,0.9611111111111112,0.0388888888888888,8 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_,crop,0.9886574074074074,0.0113425925925926,12 +ITE55351156_Passeport_Completed_,crop,0.7908496732026145,0.2091503267973855,11 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS_,crop,1.0,0.0,6 +ITE55352984_CNI_Completed_,crop,0.9977477477477475,0.0022522522522523,12 +ITE55354550_20250728_113359_Completed_,crop,0.8884615384615384,0.1115384615384615,8 +ITE55354891_image_Completed_,crop,0.975,0.025,8 +image (1)_,crop,0.9979166666666668,0.0020833333333333,12 +image (11)_,crop,0.9194444444444444,0.0805555555555556,12 +image (12)_,crop,0.8828703703703704,0.1171296296296295,12 +image (13)_,crop,0.9275462962962964,0.0724537037037036,12 +image (14)_,crop,0.9,0.0999999999999999,4 +image (15)_,crop,0.9012345679012346,0.0987654320987654,9 +image (16)_,crop,0.8765432098765432,0.1234567901234567,9 +image (17)_,crop,0.9882154882154882,0.0117845117845117,12 +image (18)_,crop,0.9170777542870566,0.0829222457129433,11 +image (19)_,crop,0.9027777777777778,0.0972222222222222,12 +image (2)_,crop,0.0625,0.9375,1 +image (21)_,crop,0.8056140350877193,0.1943859649122806,5 +image (22)_,crop,0.995246913580247,0.004753086419753,12 +image (23)_,crop,0.7727842809364549,0.2272157190635451,8 +image (24)_,crop,0.0461538461538461,0.953846153846154,5 +image (25)_,crop,1.0,0.0,8 +image (26)_,crop,0.8111111111111111,0.1888888888888888,10 +image (27)_,crop,0.8666666666666667,0.1333333333333333,9 +image (28)_,crop,0.9666666666666668,0.0333333333333333,8 +image (29)_,crop,0.9122007722007724,0.0877992277992276,10 +image (3)_,crop,0.7485260770975058,0.2514739229024941,7 +image (30)_,crop,0.8637037037037036,0.1362962962962963,10 +image (31)_,crop,0.8541666666666666,0.1458333333333333,12 +image (4)_,crop,0.8035714285714286,0.1964285714285714,8 +image (6)_,crop,0.0,0.999999999,12 +image (8)_,crop,0.9818452380952382,0.0181547619047618,12 +image (9)_,crop,0.858152958152958,0.141847041847042,11 +image_,crop,0.8542229012817248,0.1457770987182751,7 +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed_,crop_shadow_cache,0.9965277777777778,0.0034722222222222,12 +ITE55340318_ID_kat__1__Completed_,crop_shadow_cache,0.994498556998557,0.0055014430014429,12 +ITE55340822_passeport_pascal_2028_Completed_,crop_shadow_cache,0.8776094276094275,0.1223905723905725,12 +ITE55341271_C_I_face_Completed_,crop_shadow_cache,0.802415458937198,0.197584541062802,10 +ITE55346966_17537774579547958575370370624241_Completed_,crop_shadow_cache,0.6485470085470085,0.3514529914529914,5 +ITE55347865_Snapchat-715567440_Completed_,crop_shadow_cache,0.7700191570881226,0.2299808429118773,8 +ITE55347866_Snapchat-1551171803_Completed_,crop_shadow_cache,0.689240991566573,0.3107590084334269,5 +ITE55347926_DHONDT_CNI_,crop_shadow_cache,0.9012345679012346,0.0987654320987654,9 +ITE55348016_image_Completed_,crop_shadow_cache,0.975,0.025,8 +ITE55348464_IMG-20250721-WA0000_Completed_,crop_shadow_cache,0.8239538239538239,0.1760461760461761,11 +ITE55348878_CNI_RECTO_,crop_shadow_cache,0.9637408088235294,0.0362591911764705,8 +ITE55349793_2_em_CNI_CHASTAING__,crop_shadow_cache,0.988095238095238,0.0119047619047618,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS_,crop_shadow_cache,0.8428571428571429,0.1571428571428571,7 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS_,crop_shadow_cache,0.961111111111111,0.0388888888888889,8 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_,crop_shadow_cache,0.9861111111111112,0.0138888888888888,12 +ITE55351156_Passeport_Completed_,crop_shadow_cache,0.7908496732026145,0.2091503267973855,11 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS_,crop_shadow_cache,1.0,0.0,6 +ITE55352984_CNI_Completed_,crop_shadow_cache,0.9886363636363636,0.0113636363636363,12 +ITE55354550_20250728_113359_Completed_,crop_shadow_cache,0.975,0.025,8 +ITE55354891_image_Completed_,crop_shadow_cache,0.975,0.025,8 +image (1)_,crop_shadow_cache,0.9979166666666668,0.0020833333333333,12 +image (11)_,crop_shadow_cache,0.9244623655913978,0.0755376344086021,12 +image (12)_,crop_shadow_cache,0.949537037037037,0.050462962962963,12 +image (13)_,crop_shadow_cache,0.980324074074074,0.0196759259259259,12 +image (14)_,crop_shadow_cache,0.9,0.0999999999999999,4 +image (15)_,crop_shadow_cache,0.9012345679012346,0.0987654320987654,9 +image (16)_,crop_shadow_cache,0.8765432098765432,0.1234567901234567,9 +image (17)_,crop_shadow_cache,0.9856902356902356,0.0143097643097642,12 +image (18)_,crop_shadow_cache,0.9170777542870566,0.0829222457129433,11 +image (19)_,crop_shadow_cache,0.9907407407407408,0.0092592592592591,12 +image (2)_,crop_shadow_cache,0.0625,0.9375,1 +image (21)_,crop_shadow_cache,0.5623423423423424,0.4376576576576576,5 +image (22)_,crop_shadow_cache,0.995246913580247,0.004753086419753,12 +image (23)_,crop_shadow_cache,0.7727842809364549,0.2272157190635451,8 +image (24)_,crop_shadow_cache,0.0461538461538461,0.953846153846154,5 +image (25)_,crop_shadow_cache,1.0,0.0,8 +image (26)_,crop_shadow_cache,0.7252287581699346,0.2747712418300654,10 +image (27)_,crop_shadow_cache,0.8666666666666667,0.1333333333333333,9 +image (28)_,crop_shadow_cache,0.7444444444444445,0.2555555555555555,8 +image (29)_,crop_shadow_cache,0.8726839826839827,0.1273160173160172,10 +image (3)_,crop_shadow_cache,0.7539682539682538,0.2460317460317461,7 +image (30)_,crop_shadow_cache,0.8637037037037036,0.1362962962962963,10 +image (31)_,crop_shadow_cache,0.8631205673758865,0.1368794326241135,12 +image (4)_,crop_shadow_cache,0.8427083333333334,0.1572916666666666,8 +image (6)_,crop_shadow_cache,0.0,0.999999999,12 +image (8)_,crop_shadow_cache,0.9339285714285714,0.0660714285714285,12 +image (9)_,crop_shadow_cache,0.858152958152958,0.141847041847042,11 +image_,crop_shadow_cache,0.8381574852163087,0.1618425147836912,7 +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed_,crop_shadow_paddle,1.0,0.0,12 +ITE55340318_ID_kat__1__Completed_,crop_shadow_paddle,0.9858630952380952,0.0141369047619047,12 +ITE55340822_passeport_pascal_2028_Completed_,crop_shadow_paddle,0.9924242424242424,0.0075757575757576,12 +ITE55341271_C_I_face_Completed_,crop_shadow_paddle,0.8111111111111111,0.1888888888888888,10 +ITE55346966_17537774579547958575370370624241_Completed_,crop_shadow_paddle,0.9155555555555556,0.0844444444444445,5 +ITE55347865_Snapchat-715567440_Completed_,crop_shadow_paddle,0.8394636015325672,0.1605363984674328,8 +ITE55347866_Snapchat-1551171803_Completed_,crop_shadow_paddle,0.8124916943521594,0.1875083056478406,5 +ITE55347926_DHONDT_CNI_,crop_shadow_paddle,0.6790123456790123,0.3209876543209877,9 +ITE55348016_image_Completed_,crop_shadow_paddle,0.975,0.025,8 +ITE55348464_IMG-20250721-WA0000_Completed_,crop_shadow_paddle,0.833044733044733,0.1669552669552669,11 +ITE55348878_CNI_RECTO_,crop_shadow_paddle,0.8894060749299719,0.110593925070028,8 +ITE55349793_2_em_CNI_CHASTAING__,crop_shadow_paddle,0.988095238095238,0.0119047619047618,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS_,crop_shadow_paddle,0.9000000000000001,0.0999999999999998,7 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS_,crop_shadow_paddle,0.9861111111111112,0.0138888888888888,8 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_,crop_shadow_paddle,0.9619522084195998,0.0380477915804001,12 +ITE55351156_Passeport_Completed_,crop_shadow_paddle,0.7908496732026145,0.2091503267973855,11 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS_,crop_shadow_paddle,0.9166666666666666,0.0833333333333333,6 +ITE55352984_CNI_Completed_,crop_shadow_paddle,0.8507575757575757,0.1492424242424243,12 +ITE55354550_20250728_113359_Completed_,crop_shadow_paddle,0.9514957264957264,0.0485042735042735,8 +ITE55354891_image_Completed_,crop_shadow_paddle,0.975,0.025,8 +image (1)_,crop_shadow_paddle,0.9979166666666668,0.0020833333333333,12 +image (11)_,crop_shadow_paddle,0.9239583333333332,0.0760416666666666,12 +image (12)_,crop_shadow_paddle,0.8632066276803118,0.1367933723196882,12 +image (13)_,crop_shadow_paddle,0.987962962962963,0.0120370370370369,12 +image (14)_,crop_shadow_paddle,0.9,0.0999999999999999,4 +image (15)_,crop_shadow_paddle,0.9012345679012346,0.0987654320987654,9 +image (16)_,crop_shadow_paddle,0.8765432098765432,0.1234567901234567,9 +image (17)_,crop_shadow_paddle,0.9882154882154882,0.0117845117845117,12 +image (18)_,crop_shadow_paddle,0.9079868451961476,0.0920131548038525,11 +image (19)_,crop_shadow_paddle,0.9717967047930284,0.0282032952069716,12 +image (2)_,crop_shadow_paddle,0.0625,0.9375,1 +image (21)_,crop_shadow_paddle,0.670990990990991,0.329009009009009,5 +image (22)_,crop_shadow_paddle,0.9969135802469136,0.0030864197530863,12 +image (23)_,crop_shadow_paddle,0.9891304347826086,0.0108695652173913,8 +image (24)_,crop_shadow_paddle,0.5843928398645379,0.415607160135462,5 +image (25)_,crop_shadow_paddle,0.9107142857142856,0.0892857142857143,8 +image (26)_,crop_shadow_paddle,0.8711111111111111,0.1288888888888889,10 +image (27)_,crop_shadow_paddle,0.8666666666666667,0.1333333333333333,9 +image (28)_,crop_shadow_paddle,0.9916666666666668,0.0083333333333333,8 +image (29)_,crop_shadow_paddle,0.7705994897959184,0.2294005102040816,10 +image (3)_,crop_shadow_paddle,0.6199546485260772,0.3800453514739228,7 +image (30)_,crop_shadow_paddle,0.8711111111111111,0.1288888888888889,10 +image (31)_,crop_shadow_paddle,0.9249999999999998,0.0750000000000001,12 +image (4)_,crop_shadow_paddle,0.9285714285714286,0.0714285714285714,8 +image (6)_,crop_shadow_paddle,0.0,0.999999999,12 +image (8)_,crop_shadow_paddle,0.9898809523809524,0.0101190476190476,12 +image (9)_,crop_shadow_paddle,0.8595959595959596,0.1404040404040404,11 +image_,crop_shadow_paddle,0.8256514727102964,0.1743485272897036,7 +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed_,crop_shadow_paddle_cache,1.0,0.0,12 +ITE55340318_ID_kat__1__Completed_,crop_shadow_paddle_cache,0.9796176046176046,0.0203823953823953,12 +ITE55340822_passeport_pascal_2028_Completed_,crop_shadow_paddle_cache,1.0,0.0,12 +ITE55341271_C_I_face_Completed_,crop_shadow_paddle_cache,0.711111111111111,0.288888888888889,10 +ITE55346966_17537774579547958575370370624241_Completed_,crop_shadow_paddle_cache,0.8177777777777777,0.1822222222222222,5 +ITE55347865_Snapchat-715567440_Completed_,crop_shadow_paddle_cache,0.8255747126436782,0.1744252873563218,8 +ITE55347866_Snapchat-1551171803_Completed_,crop_shadow_paddle_cache,0.8631229235880399,0.1368770764119601,5 +ITE55347926_DHONDT_CNI_,crop_shadow_paddle_cache,0.7364197530864198,0.2635802469135802,9 +ITE55348016_image_Completed_,crop_shadow_paddle_cache,0.975,0.025,8 +ITE55348464_IMG-20250721-WA0000_Completed_,crop_shadow_paddle_cache,0.7173082173082174,0.2826917826917826,11 +ITE55348878_CNI_RECTO_,crop_shadow_paddle_cache,0.8308648459383754,0.1691351540616246,8 +ITE55349793_2_em_CNI_CHASTAING__,crop_shadow_paddle_cache,0.988095238095238,0.0119047619047618,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS_,crop_shadow_paddle_cache,0.976023976023976,0.0239760239760239,7 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS_,crop_shadow_paddle_cache,1.0,0.0,8 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_,crop_shadow_paddle_cache,0.9555555555555556,0.0444444444444444,12 +ITE55351156_Passeport_Completed_,crop_shadow_paddle_cache,0.8817587641117054,0.1182412358882946,11 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS_,crop_shadow_paddle_cache,1.0,0.0,6 +ITE55352984_CNI_Completed_,crop_shadow_paddle_cache,0.9499999999999998,0.0500000000000001,12 +ITE55354550_20250728_113359_Completed_,crop_shadow_paddle_cache,0.961111111111111,0.0388888888888889,8 +ITE55354891_image_Completed_,crop_shadow_paddle_cache,0.975,0.025,8 +image (1)_,crop_shadow_paddle_cache,0.9979166666666668,0.0020833333333333,12 +image (11)_,crop_shadow_paddle_cache,0.9046875,0.0953125,12 +image (12)_,crop_shadow_paddle_cache,0.8842592592592592,0.1157407407407408,12 +image (13)_,crop_shadow_paddle_cache,0.9275462962962964,0.0724537037037036,12 +image (14)_,crop_shadow_paddle_cache,0.9,0.0999999999999999,4 +image (15)_,crop_shadow_paddle_cache,0.9012345679012346,0.0987654320987654,9 +image (16)_,crop_shadow_paddle_cache,0.854320987654321,0.145679012345679,9 +image (17)_,crop_shadow_paddle_cache,0.9882154882154882,0.0117845117845117,12 +image (18)_,crop_shadow_paddle_cache,0.9170777542870566,0.0829222457129433,11 +image (19)_,crop_shadow_paddle_cache,0.9907407407407408,0.0092592592592591,12 +image (2)_,crop_shadow_paddle_cache,0.0625,0.9375,1 +image (21)_,crop_shadow_paddle_cache,0.670990990990991,0.329009009009009,5 +image (22)_,crop_shadow_paddle_cache,0.9969135802469136,0.0030864197530863,12 +image (23)_,crop_shadow_paddle_cache,0.9891304347826086,0.0108695652173913,8 +image (24)_,crop_shadow_paddle_cache,0.2348330914368649,0.765166908563135,5 +image (25)_,crop_shadow_paddle_cache,0.9107142857142856,0.0892857142857143,8 +image (26)_,crop_shadow_paddle_cache,0.82,0.18,10 +image (27)_,crop_shadow_paddle_cache,0.8666666666666667,0.1333333333333333,9 +image (28)_,crop_shadow_paddle_cache,1.0,0.0,8 +image (29)_,crop_shadow_paddle_cache,0.9726839826839828,0.0273160173160172,10 +image (3)_,crop_shadow_paddle_cache,0.609750566893424,0.390249433106576,7 +image (30)_,crop_shadow_paddle_cache,0.8711111111111111,0.1288888888888889,10 +image (31)_,crop_shadow_paddle_cache,0.6940972222222221,0.3059027777777778,12 +image (4)_,crop_shadow_paddle_cache,0.9035714285714286,0.0964285714285714,8 +image (6)_,crop_shadow_paddle_cache,0.0,0.999999999,12 +image (8)_,crop_shadow_paddle_cache,0.9825091575091576,0.0174908424908424,12 +image (9)_,crop_shadow_paddle_cache,0.853102453102453,0.1468975468975469,11 +image_,crop_shadow_paddle_cache,0.8408029878618114,0.1591970121381886,7 +image (22),no_preprocessing,0.82125,0.1787499999999999,12 +image (16),no_preprocessing,0.8765432098765432,0.1234567901234567,9 +image (18),no_preprocessing,0.4974404148822753,0.5025595851177247,11 +image (3),no_preprocessing,0.7913832199546486,0.2086167800453514,7 +ITE55348016_image_Completed,no_preprocessing,0.7644230769230769,0.2355769230769231,8 +ITE55348878_CNI_RECTO,no_preprocessing,0.8073553599071208,0.1926446400928791,8 +image (26),no_preprocessing,0.7659498207885305,0.2340501792114695,10 +image (28),no_preprocessing,0.7822072072072072,0.2177927927927928,8 +ITE55347926_DHONDT_CNI,no_preprocessing,0.6816239316239316,0.3183760683760683,9 +image (15),no_preprocessing,0.9012345679012346,0.0987654320987654,9 +ITE55354891_image_Completed,no_preprocessing,0.6394230769230769,0.3605769230769231,8 +image (4),no_preprocessing,0.975,0.025,8 +ITE55351156_Passeport_Completed,no_preprocessing,0.8813131313131314,0.1186868686868686,11 +ITE55347866_Snapchat-1551171803_Completed,no_preprocessing,0.5564221824686941,0.4435778175313059,5 +ITE55341271_C_I_face_Completed,no_preprocessing,0.5861416361416361,0.4138583638583639,10 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,no_preprocessing,0.6481481481481481,0.3518518518518518,6 +image (9),no_preprocessing,0.914141414141414,0.0858585858585858,11 +image (25),no_preprocessing,0.8667929292929293,0.1332070707070707,8 +ITE55352984_CNI_Completed,no_preprocessing,0.7075757575757576,0.2924242424242423,12 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,no_preprocessing,0.9527777777777776,0.0472222222222222,8 +ITE55349793_2_em_CNI_CHASTAING_,no_preprocessing,0.938095238095238,0.061904761904762,12 +ITE55354550_20250728_113359_Completed,no_preprocessing,0.5625,0.4375,8 +ITE55340318_ID_kat__1__Completed,no_preprocessing,0.3825320512820513,0.6174679487179486,12 +ITE55340822_passeport_pascal_2028_Completed,no_preprocessing,0.8164983164983166,0.1835016835016834,12 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,no_preprocessing,0.6349206349206349,0.3650793650793651,12 +image (2),no_preprocessing,0.0625,0.9375,1 +image (21),no_preprocessing,0.7452631578947368,0.2547368421052631,5 +image (12),no_preprocessing,0.8540820231996702,0.1459179768003298,12 +image (24),no_preprocessing,0.0461538461538461,0.953846153846154,5 +image (31),no_preprocessing,0.8666666666666666,0.1333333333333334,12 +ITE55347865_Snapchat-715567440_Completed,no_preprocessing,0.5144230769230769,0.4855769230769231,8 +image (30),no_preprocessing,0.8037037037037036,0.1962962962962964,10 +image (27),no_preprocessing,0.8888888888888888,0.1111111111111111,9 +ITE55348464_IMG-20250721-WA0000_Completed,no_preprocessing,0.8966810966810965,0.1033189033189034,11 +image (19),no_preprocessing,0.6095515276549759,0.3904484723450241,12 +ITE55346966_17537774579547958575370370624241_Completed,no_preprocessing,0.4266365007541478,0.5733634992458522,5 +image (8),no_preprocessing,0.6822344322344321,0.3177655677655678,12 +image (13),no_preprocessing,0.9907407407407408,0.0092592592592591,12 +image (23),no_preprocessing,0.0288461538461538,0.971153846153846,8 +image (1),no_preprocessing,0.6418574481074482,0.3581425518925518,12 +image (14),no_preprocessing,1.0,0.0,4 +image_,no_preprocessing,0.7751322751322751,0.2248677248677248,7 +image (11),no_preprocessing,0.3424526862026862,0.6575473137973138,12 +image (6),no_preprocessing,0.9325913349682452,0.0674086650317549,12 +image (29),no_preprocessing,0.8412925170068026,0.1587074829931973,10 +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,no_preprocessing,1.0,0.0,12 +image (17),no_preprocessing,0.6437516187516187,0.3562483812483813,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,no_preprocessing,0.1428571428571428,0.8571428571428572,7 diff --git a/data/task/cni/inference_out/combined_anls/pipeline/pipeline_means.csv b/data/task/cni/inference_out/combined_anls/pipeline/pipeline_means.csv new file mode 100644 index 0000000..1432c44 --- /dev/null +++ b/data/task/cni/inference_out/combined_anls/pipeline/pipeline_means.csv @@ -0,0 +1,6 @@ +method,raw_mean,deqa_mean,human_mean,raw_count,deqa_count,human_count +crop,0.14311980666931925,0.113280480384857,0.10987469285842993,48,48,48 +crop_shadow_cache,0.16717154207835927,0.12110082758738305,0.11779958008485086,48,48,48 +crop_shadow_paddle,0.14359221458380786,0.09942455054893862,0.09592340843496948,48,48,48 +crop_shadow_paddle_cache,0.15229752473367092,0.11032137012597554,0.10660814625695587,48,48,48 +no_preprocessing,0.3023333345414223,0.1630725097903042,0.13890361606290755,48,48,48 diff --git a/data/task/cni/inference_out/crop/field_metrics.csv b/data/task/cni/inference_out/crop/field_metrics.csv new file mode 100644 index 0000000..09b4eab --- /dev/null +++ b/data/task/cni/inference_out/crop/field_metrics.csv @@ -0,0 +1,16 @@ +field,tp,fp,fn,precision,recall,f1,accuracy +address,21,4,6,0.840000,0.777778,0.807692,0.777778 +birth_date,35,3,4,0.921053,0.897436,0.909091,0.897436 +birth_place,34,0,1,1.000000,0.971429,0.985507,0.971429 +doc_number,33,2,3,0.942857,0.916667,0.929577,0.916667 +document_type,43,4,5,0.914894,0.895833,0.905263,0.895833 +expiry_date,31,3,5,0.911765,0.861111,0.885714,0.861111 +gender,37,0,1,1.000000,0.973684,0.986667,0.973684 +issue_date,26,3,5,0.896552,0.838710,0.866667,0.838710 +issue_place,18,2,4,0.900000,0.818182,0.857143,0.818182 +name,40,1,2,0.975610,0.952381,0.963855,0.952381 +nationality,39,0,1,1.000000,0.975000,0.987342,0.975000 +permit_type,1,0,0,1.000000,1.000000,1.000000,1.000000 +personal_number,1,4,5,0.200000,0.166667,0.181818,0.166667 +remarks,1,0,0,1.000000,1.000000,1.000000,1.000000 +surname,39,2,3,0.951220,0.928571,0.939759,0.928571 diff --git a/data/task/cni/inference_out/crop/per_image_anls.csv b/data/task/cni/inference_out/crop/per_image_anls.csv new file mode 100644 index 0000000..e866478 --- /dev/null +++ b/data/task/cni/inference_out/crop/per_image_anls.csv @@ -0,0 +1,49 @@ +image,anls,hallucination_score,num_fields +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed_,1.0,0.0,12 +ITE55340318_ID_kat__1__Completed_,0.9200562169312169,0.07994378306878314,12 +ITE55340822_passeport_pascal_2028_Completed_,1.0,0.0,12 +ITE55341271_C_I_face_Completed_,0.802415458937198,0.197584541062802,10 +ITE55346966_17537774579547958575370370624241_Completed_,0.8177777777777777,0.18222222222222229,5 +ITE55347865_Snapchat-715567440_Completed_,0.9861111111111112,0.01388888888888884,8 +ITE55347866_Snapchat-1551171803_Completed_,0.9384717607973421,0.06152823920265793,5 +ITE55347926_DHONDT_CNI_,0.9012345679012346,0.09876543209876543,9 +ITE55348016_image_Completed_,0.975,0.025000000000000022,8 +ITE55348464_IMG-20250721-WA0000_Completed_,0.833044733044733,0.16695526695526697,11 +ITE55348878_CNI_RECTO_,0.9637408088235294,0.036259191176470584,8 +ITE55349793_2_em_CNI_CHASTAING__,0.9880952380952381,0.011904761904761862,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS_,0.9857142857142858,0.014285714285714235,7 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS_,0.9611111111111111,0.03888888888888886,8 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_,0.9886574074074074,0.011342592592592626,12 +ITE55351156_Passeport_Completed_,0.7908496732026145,0.20915032679738554,11 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS_,1.0,0.0,6 +ITE55352984_CNI_Completed_,0.9977477477477477,0.0022522522522523403,12 +ITE55354550_20250728_113359_Completed_,0.8884615384615384,0.11153846153846159,8 +ITE55354891_image_Completed_,0.975,0.025000000000000022,8 +image (1)_,0.9979166666666667,0.002083333333333326,12 +image (11)_,0.9194444444444444,0.0805555555555556,12 +image (12)_,0.8828703703703704,0.11712962962962958,12 +image (13)_,0.9275462962962964,0.07245370370370363,12 +image (14)_,0.9,0.09999999999999998,4 +image (15)_,0.9012345679012346,0.09876543209876543,9 +image (16)_,0.8765432098765432,0.12345679012345678,9 +image (17)_,0.9882154882154882,0.011784511784511786,12 +image (18)_,0.9170777542870566,0.08292224571294338,11 +image (19)_,0.9027777777777778,0.09722222222222221,12 +image (2)_,0.0625,0.9375,1 +image (21)_,0.8056140350877193,0.19438596491228066,5 +image (22)_,0.995246913580247,0.004753086419753028,12 +image (23)_,0.7727842809364549,0.2272157190635451,8 +image (24)_,0.04615384615384614,0.9538461538461539,5 +image (25)_,1.0,0.0,8 +image (26)_,0.8111111111111111,0.18888888888888888,10 +image (27)_,0.8666666666666667,0.1333333333333333,9 +image (28)_,0.9666666666666667,0.033333333333333326,8 +image (29)_,0.9122007722007723,0.08779922779922766,10 +image (3)_,0.7485260770975058,0.25147392290249415,7 +image (30)_,0.8637037037037036,0.13629629629629636,10 +image (31)_,0.8541666666666666,0.14583333333333337,12 +image (4)_,0.8035714285714286,0.1964285714285714,8 +image (6)_,0.0,0.999999999,12 +image (8)_,0.9818452380952382,0.01815476190476184,12 +image (9)_,0.858152958152958,0.141847041847042,11 +image_,0.8542229012817248,0.14577709871827516,7 diff --git a/data/task/cni/inference_out/crop_shadow_cache/field_metrics.csv b/data/task/cni/inference_out/crop_shadow_cache/field_metrics.csv new file mode 100644 index 0000000..599aa2e --- /dev/null +++ b/data/task/cni/inference_out/crop_shadow_cache/field_metrics.csv @@ -0,0 +1,16 @@ +field,tp,fp,fn,precision,recall,f1,accuracy +address,20,5,7,0.800000,0.740741,0.769231,0.740741 +birth_date,35,3,4,0.921053,0.897436,0.909091,0.897436 +birth_place,33,1,2,0.970588,0.942857,0.956522,0.942857 +doc_number,33,2,3,0.942857,0.916667,0.929577,0.916667 +document_type,41,6,7,0.872340,0.854167,0.863158,0.854167 +expiry_date,31,3,5,0.911765,0.861111,0.885714,0.861111 +gender,36,1,2,0.972973,0.947368,0.960000,0.947368 +issue_date,27,2,4,0.931034,0.870968,0.900000,0.870968 +issue_place,17,3,5,0.850000,0.772727,0.809524,0.772727 +name,41,0,1,1.000000,0.976190,0.987952,0.976190 +nationality,38,1,2,0.974359,0.950000,0.962025,0.950000 +permit_type,1,0,0,1.000000,1.000000,1.000000,1.000000 +personal_number,1,4,5,0.200000,0.166667,0.181818,0.166667 +remarks,1,0,0,1.000000,1.000000,1.000000,1.000000 +surname,38,3,4,0.926829,0.904762,0.915663,0.904762 diff --git a/data/task/cni/inference_out/crop_shadow_cache/per_image_anls.csv b/data/task/cni/inference_out/crop_shadow_cache/per_image_anls.csv new file mode 100644 index 0000000..98f9d7c --- /dev/null +++ b/data/task/cni/inference_out/crop_shadow_cache/per_image_anls.csv @@ -0,0 +1,49 @@ +image,anls,hallucination_score,num_fields +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed_,0.9965277777777778,0.00347222222222221,12 +ITE55340318_ID_kat__1__Completed_,0.994498556998557,0.005501443001442952,12 +ITE55340822_passeport_pascal_2028_Completed_,0.8776094276094275,0.1223905723905725,12 +ITE55341271_C_I_face_Completed_,0.802415458937198,0.197584541062802,10 +ITE55346966_17537774579547958575370370624241_Completed_,0.6485470085470085,0.35145299145299147,5 +ITE55347865_Snapchat-715567440_Completed_,0.7700191570881226,0.22998084291187737,8 +ITE55347866_Snapchat-1551171803_Completed_,0.689240991566573,0.31075900843342696,5 +ITE55347926_DHONDT_CNI_,0.9012345679012346,0.09876543209876543,9 +ITE55348016_image_Completed_,0.975,0.025000000000000022,8 +ITE55348464_IMG-20250721-WA0000_Completed_,0.8239538239538239,0.17604617604617612,11 +ITE55348878_CNI_RECTO_,0.9637408088235294,0.036259191176470584,8 +ITE55349793_2_em_CNI_CHASTAING__,0.9880952380952381,0.011904761904761862,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS_,0.8428571428571429,0.15714285714285714,7 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS_,0.961111111111111,0.03888888888888897,8 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_,0.9861111111111112,0.01388888888888884,12 +ITE55351156_Passeport_Completed_,0.7908496732026145,0.20915032679738554,11 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS_,1.0,0.0,6 +ITE55352984_CNI_Completed_,0.9886363636363636,0.011363636363636354,12 +ITE55354550_20250728_113359_Completed_,0.975,0.025000000000000022,8 +ITE55354891_image_Completed_,0.975,0.025000000000000022,8 +image (1)_,0.9979166666666667,0.002083333333333326,12 +image (11)_,0.9244623655913978,0.07553763440860217,12 +image (12)_,0.949537037037037,0.05046296296296304,12 +image (13)_,0.9803240740740741,0.01967592592592593,12 +image (14)_,0.9,0.09999999999999998,4 +image (15)_,0.9012345679012346,0.09876543209876543,9 +image (16)_,0.8765432098765432,0.12345679012345678,9 +image (17)_,0.9856902356902357,0.014309764309764272,12 +image (18)_,0.9170777542870566,0.08292224571294338,11 +image (19)_,0.9907407407407408,0.00925925925925919,12 +image (2)_,0.0625,0.9375,1 +image (21)_,0.5623423423423424,0.43765765765765763,5 +image (22)_,0.995246913580247,0.004753086419753028,12 +image (23)_,0.7727842809364549,0.2272157190635451,8 +image (24)_,0.04615384615384614,0.9538461538461539,5 +image (25)_,1.0,0.0,8 +image (26)_,0.7252287581699346,0.2747712418300654,10 +image (27)_,0.8666666666666667,0.1333333333333333,9 +image (28)_,0.7444444444444445,0.25555555555555554,8 +image (29)_,0.8726839826839827,0.12731601731601727,10 +image (3)_,0.7539682539682538,0.24603174603174616,7 +image (30)_,0.8637037037037036,0.13629629629629636,10 +image (31)_,0.8631205673758865,0.1368794326241135,12 +image (4)_,0.8427083333333334,0.1572916666666666,8 +image (6)_,0.0,0.999999999,12 +image (8)_,0.9339285714285714,0.06607142857142856,12 +image (9)_,0.858152958152958,0.141847041847042,11 +image_,0.8381574852163087,0.16184251478369127,7 diff --git a/data/task/cni/inference_out/crop_shadow_paddle/field_metrics.csv b/data/task/cni/inference_out/crop_shadow_paddle/field_metrics.csv new file mode 100644 index 0000000..6837cb7 --- /dev/null +++ b/data/task/cni/inference_out/crop_shadow_paddle/field_metrics.csv @@ -0,0 +1,16 @@ +field,tp,fp,fn,precision,recall,f1,accuracy +address,21,5,6,0.807692,0.777778,0.792453,0.777778 +birth_date,36,2,3,0.947368,0.923077,0.935065,0.923077 +birth_place,34,0,1,1.000000,0.971429,0.985507,0.971429 +doc_number,32,3,4,0.914286,0.888889,0.901408,0.888889 +document_type,44,3,4,0.936170,0.916667,0.926316,0.916667 +expiry_date,32,2,4,0.941176,0.888889,0.914286,0.888889 +gender,37,0,1,1.000000,0.973684,0.986667,0.973684 +issue_date,24,6,7,0.800000,0.774194,0.786885,0.774194 +issue_place,19,2,3,0.904762,0.863636,0.883721,0.863636 +name,39,2,3,0.951220,0.928571,0.939759,0.928571 +nationality,39,0,1,1.000000,0.975000,0.987342,0.975000 +permit_type,1,0,0,1.000000,1.000000,1.000000,1.000000 +personal_number,2,3,4,0.400000,0.333333,0.363636,0.333333 +remarks,0,1,1,0.000000,0.000000,0.000000,0.000000 +surname,37,4,5,0.902439,0.880952,0.891566,0.880952 diff --git a/data/task/cni/inference_out/crop_shadow_paddle/per_image_anls.csv b/data/task/cni/inference_out/crop_shadow_paddle/per_image_anls.csv new file mode 100644 index 0000000..08b2dc6 --- /dev/null +++ b/data/task/cni/inference_out/crop_shadow_paddle/per_image_anls.csv @@ -0,0 +1,49 @@ +image,anls,hallucination_score,num_fields +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed_,1.0,0.0,12 +ITE55340318_ID_kat__1__Completed_,0.9858630952380952,0.014136904761904767,12 +ITE55340822_passeport_pascal_2028_Completed_,0.9924242424242423,0.00757575757575768,12 +ITE55341271_C_I_face_Completed_,0.8111111111111111,0.18888888888888888,10 +ITE55346966_17537774579547958575370370624241_Completed_,0.9155555555555555,0.08444444444444454,5 +ITE55347865_Snapchat-715567440_Completed_,0.8394636015325672,0.16053639846743284,8 +ITE55347866_Snapchat-1551171803_Completed_,0.8124916943521594,0.1875083056478406,5 +ITE55347926_DHONDT_CNI_,0.6790123456790123,0.32098765432098775,9 +ITE55348016_image_Completed_,0.975,0.025000000000000022,8 +ITE55348464_IMG-20250721-WA0000_Completed_,0.833044733044733,0.16695526695526697,11 +ITE55348878_CNI_RECTO_,0.8894060749299719,0.11059392507002808,8 +ITE55349793_2_em_CNI_CHASTAING__,0.9880952380952381,0.011904761904761862,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS_,0.9000000000000001,0.09999999999999987,7 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS_,0.9861111111111112,0.01388888888888884,8 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_,0.9619522084195998,0.03804779158040017,12 +ITE55351156_Passeport_Completed_,0.7908496732026145,0.20915032679738554,11 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS_,0.9166666666666666,0.08333333333333337,6 +ITE55352984_CNI_Completed_,0.8507575757575757,0.1492424242424243,12 +ITE55354550_20250728_113359_Completed_,0.9514957264957264,0.048504273504273554,8 +ITE55354891_image_Completed_,0.975,0.025000000000000022,8 +image (1)_,0.9979166666666667,0.002083333333333326,12 +image (11)_,0.9239583333333333,0.07604166666666667,12 +image (12)_,0.8632066276803118,0.13679337231968824,12 +image (13)_,0.987962962962963,0.012037037037036957,12 +image (14)_,0.9,0.09999999999999998,4 +image (15)_,0.9012345679012346,0.09876543209876543,9 +image (16)_,0.8765432098765432,0.12345679012345678,9 +image (17)_,0.9882154882154882,0.011784511784511786,12 +image (18)_,0.9079868451961475,0.09201315480385253,11 +image (19)_,0.9717967047930284,0.028203295206971624,12 +image (2)_,0.0625,0.9375,1 +image (21)_,0.670990990990991,0.329009009009009,5 +image (22)_,0.9969135802469137,0.0030864197530863224,12 +image (23)_,0.9891304347826086,0.010869565217391353,8 +image (24)_,0.5843928398645379,0.41560716013546206,5 +image (25)_,0.9107142857142857,0.0892857142857143,8 +image (26)_,0.8711111111111111,0.12888888888888894,10 +image (27)_,0.8666666666666667,0.1333333333333333,9 +image (28)_,0.9916666666666667,0.008333333333333304,8 +image (29)_,0.7705994897959184,0.2294005102040816,10 +image (3)_,0.6199546485260772,0.3800453514739228,7 +image (30)_,0.8711111111111111,0.12888888888888894,10 +image (31)_,0.9249999999999998,0.07500000000000018,12 +image (4)_,0.9285714285714286,0.0714285714285714,8 +image (6)_,0.0,0.999999999,12 +image (8)_,0.9898809523809523,0.010119047619047694,12 +image (9)_,0.8595959595959596,0.14040404040404042,11 +image_,0.8256514727102964,0.17434852728970363,7 diff --git a/data/task/cni/inference_out/crop_shadow_paddle_cache/field_metrics.csv b/data/task/cni/inference_out/crop_shadow_paddle_cache/field_metrics.csv new file mode 100644 index 0000000..b523f17 --- /dev/null +++ b/data/task/cni/inference_out/crop_shadow_paddle_cache/field_metrics.csv @@ -0,0 +1,16 @@ +field,tp,fp,fn,precision,recall,f1,accuracy +address,21,5,6,0.807692,0.777778,0.792453,0.777778 +birth_date,35,3,4,0.921053,0.897436,0.909091,0.897436 +birth_place,34,0,1,1.000000,0.971429,0.985507,0.971429 +doc_number,33,2,3,0.942857,0.916667,0.929577,0.916667 +document_type,44,3,4,0.936170,0.916667,0.926316,0.916667 +expiry_date,29,5,7,0.852941,0.805556,0.828571,0.805556 +gender,37,0,1,1.000000,0.973684,0.986667,0.973684 +issue_date,25,4,6,0.862069,0.806452,0.833333,0.806452 +issue_place,17,3,5,0.850000,0.772727,0.809524,0.772727 +name,36,5,6,0.878049,0.857143,0.867470,0.857143 +nationality,39,0,1,1.000000,0.975000,0.987342,0.975000 +permit_type,1,0,0,1.000000,1.000000,1.000000,1.000000 +personal_number,3,3,3,0.500000,0.500000,0.500000,0.500000 +remarks,1,0,0,1.000000,1.000000,1.000000,1.000000 +surname,37,4,5,0.902439,0.880952,0.891566,0.880952 diff --git a/data/task/cni/inference_out/crop_shadow_paddle_cache/per_image_anls.csv b/data/task/cni/inference_out/crop_shadow_paddle_cache/per_image_anls.csv new file mode 100644 index 0000000..b960357 --- /dev/null +++ b/data/task/cni/inference_out/crop_shadow_paddle_cache/per_image_anls.csv @@ -0,0 +1,49 @@ +image,anls,hallucination_score,num_fields +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed_,1.0,0.0,12 +ITE55340318_ID_kat__1__Completed_,0.9796176046176046,0.02038239538239539,12 +ITE55340822_passeport_pascal_2028_Completed_,1.0,0.0,12 +ITE55341271_C_I_face_Completed_,0.711111111111111,0.288888888888889,10 +ITE55346966_17537774579547958575370370624241_Completed_,0.8177777777777777,0.18222222222222229,5 +ITE55347865_Snapchat-715567440_Completed_,0.8255747126436782,0.1744252873563218,8 +ITE55347866_Snapchat-1551171803_Completed_,0.8631229235880399,0.13687707641196012,5 +ITE55347926_DHONDT_CNI_,0.7364197530864198,0.2635802469135802,9 +ITE55348016_image_Completed_,0.975,0.025000000000000022,8 +ITE55348464_IMG-20250721-WA0000_Completed_,0.7173082173082174,0.28269178269178263,11 +ITE55348878_CNI_RECTO_,0.8308648459383754,0.1691351540616246,8 +ITE55349793_2_em_CNI_CHASTAING__,0.9880952380952381,0.011904761904761862,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS_,0.9760239760239761,0.02397602397602394,7 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS_,1.0,0.0,8 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS_,0.9555555555555556,0.0444444444444444,12 +ITE55351156_Passeport_Completed_,0.8817587641117054,0.1182412358882946,11 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS_,1.0,0.0,6 +ITE55352984_CNI_Completed_,0.9499999999999998,0.050000000000000155,12 +ITE55354550_20250728_113359_Completed_,0.961111111111111,0.03888888888888897,8 +ITE55354891_image_Completed_,0.975,0.025000000000000022,8 +image (1)_,0.9979166666666667,0.002083333333333326,12 +image (11)_,0.9046875,0.09531250000000002,12 +image (12)_,0.8842592592592592,0.11574074074074081,12 +image (13)_,0.9275462962962964,0.07245370370370363,12 +image (14)_,0.9,0.09999999999999998,4 +image (15)_,0.9012345679012346,0.09876543209876543,9 +image (16)_,0.854320987654321,0.14567901234567904,9 +image (17)_,0.9882154882154882,0.011784511784511786,12 +image (18)_,0.9170777542870566,0.08292224571294338,11 +image (19)_,0.9907407407407408,0.00925925925925919,12 +image (2)_,0.0625,0.9375,1 +image (21)_,0.670990990990991,0.329009009009009,5 +image (22)_,0.9969135802469137,0.0030864197530863224,12 +image (23)_,0.9891304347826086,0.010869565217391353,8 +image (24)_,0.23483309143686498,0.765166908563135,5 +image (25)_,0.9107142857142857,0.0892857142857143,8 +image (26)_,0.82,0.18000000000000005,10 +image (27)_,0.8666666666666667,0.1333333333333333,9 +image (28)_,1.0,0.0,8 +image (29)_,0.9726839826839827,0.027316017316017294,10 +image (3)_,0.609750566893424,0.390249433106576,7 +image (30)_,0.8711111111111111,0.12888888888888894,10 +image (31)_,0.6940972222222221,0.30590277777777786,12 +image (4)_,0.9035714285714286,0.09642857142857142,8 +image (6)_,0.0,0.999999999,12 +image (8)_,0.9825091575091576,0.01749084249084243,12 +image (9)_,0.853102453102453,0.14689754689754697,11 +image_,0.8408029878618114,0.1591970121381886,7 diff --git a/data/task/cni/inference_out/no_preprocessing/field_metrics.csv b/data/task/cni/inference_out/no_preprocessing/field_metrics.csv new file mode 100644 index 0000000..290c062 --- /dev/null +++ b/data/task/cni/inference_out/no_preprocessing/field_metrics.csv @@ -0,0 +1,16 @@ +field,tp,fp,fn,precision,recall,f1,accuracy +address,13,6,14,0.684211,0.481481,0.565217,0.481481 +birth_date,33,4,6,0.891892,0.846154,0.868421,0.846154 +birth_place,22,2,13,0.916667,0.628571,0.745763,0.628571 +doc_number,28,5,8,0.848485,0.777778,0.811594,0.777778 +document_type,26,22,22,0.541667,0.541667,0.541667,0.541667 +expiry_date,28,5,8,0.848485,0.777778,0.811594,0.777778 +gender,34,2,4,0.944444,0.894737,0.918919,0.894737 +issue_date,20,7,11,0.740741,0.645161,0.689655,0.645161 +issue_place,6,3,16,0.666667,0.272727,0.387097,0.272727 +name,28,12,14,0.700000,0.666667,0.682927,0.666667 +nationality,33,4,7,0.891892,0.825000,0.857143,0.825000 +permit_type,1,0,0,1.000000,1.000000,1.000000,1.000000 +personal_number,1,0,5,1.000000,0.166667,0.285714,0.166667 +remarks,0,1,1,0.000000,0.000000,0.000000,0.000000 +surname,32,8,10,0.800000,0.761905,0.780488,0.761905 diff --git a/data/task/cni/inference_out/no_preprocessing/per_image_anls.csv b/data/task/cni/inference_out/no_preprocessing/per_image_anls.csv new file mode 100644 index 0000000..8b8bf03 --- /dev/null +++ b/data/task/cni/inference_out/no_preprocessing/per_image_anls.csv @@ -0,0 +1,49 @@ +image,anls,hallucination_score,num_fields +image (22),0.82125,0.17874999999999996,12 +image (16),0.8765432098765432,0.12345679012345678,9 +image (18),0.4974404148822753,0.5025595851177247,11 +image (3),0.7913832199546486,0.20861678004535145,7 +ITE55348016_image_Completed,0.7644230769230769,0.23557692307692313,8 +ITE55348878_CNI_RECTO,0.8073553599071208,0.19264464009287918,8 +image (26),0.7659498207885305,0.23405017921146953,10 +image (28),0.7822072072072072,0.21779279279279284,8 +ITE55347926_DHONDT_CNI,0.6816239316239316,0.31837606837606836,9 +image (15),0.9012345679012346,0.09876543209876543,9 +ITE55354891_image_Completed,0.6394230769230769,0.36057692307692313,8 +image (4),0.975,0.025000000000000022,8 +ITE55351156_Passeport_Completed,0.8813131313131314,0.11868686868686862,11 +ITE55347866_Snapchat-1551171803_Completed,0.5564221824686941,0.4435778175313059,5 +ITE55341271_C_I_face_Completed,0.5861416361416361,0.41385836385836394,10 +ITE55352532_PLI_INCONNU_-_ADH_625374_42297046_DMS,0.6481481481481481,0.35185185185185186,6 +image (9),0.9141414141414141,0.08585858585858586,11 +image (25),0.8667929292929293,0.13320707070707072,8 +ITE55352984_CNI_Completed,0.7075757575757576,0.29242424242424236,12 +ITE55351087_PLI_INCONNU_-_ADH_625120_42296075_DMS,0.9527777777777777,0.047222222222222276,8 +ITE55349793_2_em_CNI_CHASTAING_,0.938095238095238,0.06190476190476202,12 +ITE55354550_20250728_113359_Completed,0.5625,0.4375,8 +ITE55340318_ID_kat__1__Completed,0.3825320512820513,0.6174679487179486,12 +ITE55340822_passeport_pascal_2028_Completed,0.8164983164983166,0.1835016835016834,12 +ITE55351128_PLI_INCONNU_-_ADH_625123_42296103_DMS,0.6349206349206349,0.3650793650793651,12 +image (2),0.0625,0.9375,1 +image (21),0.7452631578947368,0.25473684210526315,5 +image (12),0.8540820231996702,0.1459179768003298,12 +image (24),0.04615384615384614,0.9538461538461539,5 +image (31),0.8666666666666666,0.13333333333333341,12 +ITE55347865_Snapchat-715567440_Completed,0.5144230769230769,0.48557692307692313,8 +image (30),0.8037037037037036,0.1962962962962964,10 +image (27),0.8888888888888888,0.11111111111111116,9 +ITE55348464_IMG-20250721-WA0000_Completed,0.8966810966810965,0.10331890331890348,11 +image (19),0.6095515276549759,0.3904484723450241,12 +ITE55346966_17537774579547958575370370624241_Completed,0.42663650075414783,0.5733634992458522,5 +image (8),0.6822344322344321,0.31776556776556786,12 +image (13),0.9907407407407408,0.00925925925925919,12 +image (23),0.02884615384615384,0.9711538461538461,8 +image (1),0.6418574481074482,0.35814255189255184,12 +image (14),1.0,0.0,4 +image_,0.7751322751322751,0.22486772486772488,7 +image (11),0.3424526862026862,0.6575473137973138,12 +image (6),0.9325913349682451,0.06740866503175491,12 +image (29),0.8412925170068026,0.15870748299319737,10 +ITE55336627_PASSEPORT_Abasse_GUEYE__Completed,1.0,0.0,12 +image (17),0.6437516187516187,0.3562483812483813,12 +ITE55350967_PLI_INCONNU_-_ADH_625302_42296806_DMS,0.14285714285714285,0.8571428571428572,7 diff --git a/scripts/compute_hallucination_from_anls.py b/scripts/compute_hallucination_from_anls.py new file mode 100644 index 0000000..1344860 --- /dev/null +++ b/scripts/compute_hallucination_from_anls.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +""" +Compute ANLS and hallucination scores from per-sample evaluation JSONs and plot results. + +Inputs: one or more JSON files with schema like: +[ + { + "image": "image (22)", + "num_pred_fields": 10, + "num_gt_fields": 12, + "num_correct": 9, + "all_correct": false, + "fields": [ + {"field": "address", "pred": "...", "gt": "...", "correct": true}, + ... + ] + }, + ... +] + +ANLS definition: average normalized Levenshtein similarity over fields present in ground truth. +Here we approximate per-field similarity as: + sim = 1 - (levenshtein_distance(pred, gt) / max(len(pred), len(gt))) + clipped into [0, 1], and treat empty max length as exact match (1.0). + +Per-image ANLS is the mean of field similarities for that image. Hallucination is 1 - ANLS. + +Outputs: +- CSV per input JSON placed next to it: per_image_anls.csv with columns [image, anls, hallucination_score, num_fields] +- PNG bar chart per input JSON: hallucination_per_image.png with mean line and title. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Dict, List, Tuple + +import pandas as pd +import matplotlib.pyplot as plt + + +def levenshtein_distance(a: str, b: str) -> int: + """Compute Levenshtein distance between two strings (iterative DP).""" + if a == b: + return 0 + if len(a) == 0: + return len(b) + if len(b) == 0: + return len(a) + previous_row = list(range(len(b) + 1)) + for i, ca in enumerate(a, start=1): + current_row = [i] + for j, cb in enumerate(b, start=1): + insertions = previous_row[j] + 1 + deletions = current_row[j - 1] + 1 + substitutions = previous_row[j - 1] + (0 if ca == cb else 1) + current_row.append(min(insertions, deletions, substitutions)) + previous_row = current_row + return previous_row[-1] + + +def normalized_similarity(pred: str, gt: str) -> float: + """Return 1 - normalized edit distance in [0, 1].""" + pred = pred or "" + gt = gt or "" + max_len = max(len(pred), len(gt)) + if max_len == 0: + return 1.0 + dist = levenshtein_distance(pred, gt) + sim = 1.0 - (dist / max_len) + if sim < 0.0: + return 0.0 + if sim > 1.0: + return 1.0 + return sim + + +def compute_anls_for_record(record: Dict) -> Tuple[float, int]: + """Compute ANLS and number of fields for a single record object.""" + fields = record.get("fields") or [] + if not isinstance(fields, list) or len(fields) == 0: + return 0.0, 0 + sims: List[float] = [] + for f in fields: + pred = str(f.get("pred", "")) + gt = str(f.get("gt", "")) + sims.append(normalized_similarity(pred, gt)) + anls = float(sum(sims) / len(sims)) if sims else 0.0 + return anls, len(sims) + + +def process_json(json_path: Path) -> Path: + with json_path.open("r", encoding="utf-8") as f: + data = json.load(f) + + rows = [] + for rec in data: + image_name = rec.get("image") + anls, num_fields = compute_anls_for_record(rec) + hallucination = 1.0 - anls + rows.append({ + "image": image_name, + "anls": anls, + "hallucination_score": hallucination, + "num_fields": int(num_fields), + }) + + df = pd.DataFrame(rows) + out_csv = json_path.parent / "per_image_anls.csv" + df.to_csv(out_csv, index=False) + + # Plot hallucination bar chart with mean line + if len(df) > 0: + sorted_df = df.sort_values("hallucination_score", ascending=False).reset_index(drop=True) + plt.figure(figsize=(max(8, len(sorted_df) * 0.12), 5)) + plt.bar(range(len(sorted_df)), sorted_df["hallucination_score"].values, color="#1f77b4") + mean_val = float(sorted_df["hallucination_score"].mean()) + plt.axhline(mean_val, color="red", linestyle="--", label=f"Mean={mean_val:.3f}") + plt.xlabel("Image (sorted by hallucination)") + plt.ylabel("Hallucination = 1 - ANLS") + plt.title(f"Hallucination per image: {json_path.parent.name}") + plt.legend() + plt.tight_layout() + out_png = json_path.parent / "hallucination_per_image.png" + plt.savefig(out_png, dpi=150) + plt.close() + + return out_csv + + +def common_parent(paths: List[Path]) -> Path: + if not paths: + return Path.cwd() + common = Path(Path(paths[0]).anchor) + parts = list(Path(paths[0]).resolve().parts) + for i in range(1, len(paths)): + other_parts = list(Path(paths[i]).resolve().parts) + # shrink parts to common prefix + new_parts: List[str] = [] + for a, b in zip(parts, other_parts): + if a == b: + new_parts.append(a) + else: + break + parts = new_parts + if not parts: + return Path.cwd() + return Path(*parts) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compute ANLS and hallucination from per-sample JSONs and plot results.") + parser.add_argument("inputs", nargs="+", help="Paths to per_sample_eval.json files") + args = parser.parse_args() + + any_error = False + combined_rows: List[Dict] = [] + input_paths: List[Path] = [] + for in_path_str in args.inputs: + path = Path(in_path_str) + if not path.exists(): + print(f"[WARN] File does not exist: {path}", file=sys.stderr) + any_error = True + continue + try: + out_csv = process_json(path) + print(f"Processed: {path} -> {out_csv}") + # Load just-written CSV to aggregate and tag method + df = pd.read_csv(out_csv) + method_name = path.parent.name + df["method"] = method_name + combined_rows.extend(df.to_dict(orient="records")) + input_paths.append(path) + except Exception as exc: + print(f"[ERROR] Failed to process {path}: {exc}", file=sys.stderr) + any_error = True + # Create combined outputs if we have multiple inputs + if combined_rows: + combo_df = pd.DataFrame(combined_rows) + # Reorder columns + cols = ["image", "method", "anls", "hallucination_score", "num_fields"] + combo_df = combo_df[cols] + base_outdir = common_parent(input_paths) + combined_dir = base_outdir / "combined_anls" + combined_dir.mkdir(parents=True, exist_ok=True) + combined_csv = combined_dir / "combined_per_image_anls.csv" + combo_df.to_csv(combined_csv, index=False) + + # Mean hallucination per method (bar chart) + means = combo_df.groupby("method")["hallucination_score"].mean().sort_values(ascending=False) + stds = combo_df.groupby("method")["hallucination_score"].std().reindex(means.index) + plt.figure(figsize=(max(6, len(means) * 1.2), 5)) + plt.bar(means.index, means.values, yerr=stds.values, capsize=4, color="#2ca02c") + overall_mean = float(combo_df["hallucination_score"].mean()) + plt.axhline(overall_mean, color="red", linestyle="--", label=f"Overall mean={overall_mean:.3f}") + plt.ylabel("Mean hallucination (1 - ANLS)") + plt.title("Mean hallucination by method") + plt.xticks(rotation=20, ha="right") + plt.legend() + plt.tight_layout() + bar_png = combined_dir / "mean_hallucination_by_method.png" + plt.savefig(bar_png, dpi=160) + plt.close() + + # Heatmap: images x methods (hallucination) + pivot = combo_df.pivot_table(index="image", columns="method", values="hallucination_score", aggfunc="mean") + # Sort images by average hallucination descending for readability + pivot = pivot.reindex(pivot.mean(axis=1).sort_values(ascending=False).index) + plt.figure(figsize=(max(8, len(pivot.columns) * 1.0), max(6, len(pivot.index) * 0.25))) + im = plt.imshow(pivot.values, aspect="auto", cmap="viridis") + plt.colorbar(im, label="Hallucination (1 - ANLS)") + plt.xticks(range(len(pivot.columns)), pivot.columns, rotation=30, ha="right") + plt.yticks(range(len(pivot.index)), pivot.index) + plt.title("Hallucination per image across methods") + plt.tight_layout() + heatmap_png = combined_dir / "hallucination_heatmap.png" + plt.savefig(heatmap_png, dpi=160) + plt.close() + + print(f"Combined CSV: {combined_csv}") + print(f"Saved: {bar_png}") + print(f"Saved: {heatmap_png}") + + # Line chart: 1 line per method over images, hide image names + # Use same image order as pivot + methods = list(pivot.columns) + x = list(range(len(pivot.index))) + plt.figure(figsize=(max(10, len(x) * 0.12), 5)) + colors = plt.rcParams['axes.prop_cycle'].by_key().get('color', ['#1f77b4','#ff7f0e','#2ca02c','#d62728','#9467bd']) + for idx, method in enumerate(methods): + y = pivot[method].to_numpy() + plt.plot(x, y, label=method, linewidth=1.8, color=colors[idx % len(colors)]) + plt.ylim(0.0, 1.0) + plt.xlabel("Images (sorted by overall hallucination)") + plt.ylabel("Hallucination (1 - ANLS)") + plt.title("Hallucination across images by method") + plt.xticks([], []) # hide image names + # Mean note box + mean_lines = [] + for method in methods: + m = float(combo_df[combo_df["method"] == method]["hallucination_score"].mean()) + mean_lines.append(f"{method}: {m:.3f}") + text = "\n".join(mean_lines) + plt.gca().text(0.99, 0.01, text, transform=plt.gca().transAxes, + fontsize=9, va='bottom', ha='right', + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8, edgecolor='gray')) + plt.legend(loc="upper right", ncol=min(3, len(methods))) + plt.tight_layout() + line_png = combined_dir / "hallucination_lines_by_method.png" + plt.savefig(line_png, dpi=160) + plt.close() + + # Grouped-by-image interlocking line chart with image labels + # Build a consistent x position per image, with small offsets per method + base_x = list(range(len(pivot.index))) + offsets = { + m: ((i - (len(methods) - 1) / 2) * 0.12) for i, m in enumerate(methods) + } + # Cap width to avoid extremely long images; dynamic but limited + width = min(16, max(10, len(base_x) * 0.12)) + plt.figure(figsize=(width, 6)) + for idx, method in enumerate(methods): + # Fill missing values with 0 to connect lines seamlessly + y = pivot[method].fillna(0.0).to_numpy() + x_shifted = [bx + offsets[method] for bx in base_x] + plt.plot(x_shifted, y, label=method, linewidth=1.8, marker='o', markersize=3, + color=colors[idx % len(colors)]) + plt.ylim(0.0, 1.0) + plt.xlim(-0.5, len(base_x) - 0.5) + # Hide image names; keep index ticks sparse for readability + plt.xticks([], []) + plt.xlabel("Images (index)") + plt.ylabel("Hallucination (1 - ANLS)") + plt.title("Hallucination by image (interlocked methods)") + plt.grid(axis='y', linestyle='--', alpha=0.3) + # Add box with per-method mean + text2 = "\n".join([f"{m}: {float(combo_df[combo_df['method']==m]['hallucination_score'].mean()):.3f}" for m in methods]) + plt.gca().text(0.99, 0.01, text2, transform=plt.gca().transAxes, + fontsize=9, va='bottom', ha='right', + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8, edgecolor='gray')) + plt.legend(loc='upper right', ncol=min(3, len(methods))) + plt.tight_layout() + group_line_png = combined_dir / "hallucination_interlocked_by_image.png" + plt.savefig(group_line_png, dpi=160) + plt.close() + + print(f"Combined CSV: {combined_csv}") + print(f"Saved: {bar_png}") + print(f"Saved: {heatmap_png}") + print(f"Saved: {line_png}") + print(f"Saved: {group_line_png}") + + if any_error: + sys.exit(1) + + +if __name__ == "__main__": + main() + + diff --git a/scripts/pipeline_compare.py b/scripts/pipeline_compare.py new file mode 100644 index 0000000..cea7b55 --- /dev/null +++ b/scripts/pipeline_compare.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +""" +Compare hallucination across three pipelines over five preprocessing methods: + 1) Raw: all images + 2) DeQA-filtered: keep images with DeQA score >= threshold (default 2.6) + 3) Human-filtered: keep images labeled High in CSV labels + +Inputs: + - One or more per_sample_eval.json files (or per_image_anls.csv already generated) + - DeQA score file (txt): lines like "3.9 - image (9)_0.png" + - Human labels CSV with columns: filename,label where label in {High,Low} + +Outputs: + - Combined means CSV: method vs mean hallucination for each pipeline + - Line chart (3 lines): hallucination mean per method across the three pipelines +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +from pathlib import Path +from typing import Dict, List, Tuple, Optional + +import pandas as pd +import matplotlib.pyplot as plt + + +def canonical_key(name: str) -> str: + """Map various filenames to a canonical key used by per_sample_eval 'image' field. + + Examples: + - "image (9)_0.png" -> "image (9)" + - "image (22)" -> "image (22)" + - "foo/bar/image (15)_3.jpg" -> "image (15)" + - other names -> stem without extension + """ + if not name: + return name + # Keep only basename + base = Path(name).name + # Try pattern image (N) + m = re.search(r"(image \(\d+\))", base, flags=re.IGNORECASE) + if m: + return m.group(1) + # Fallback: remove extension + return Path(base).stem + + +def read_deqa_scores(txt_path: Path) -> Dict[str, float]: + scores: Dict[str, float] = {} + with txt_path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + # Accept formats: "3.9 - filename" or "filename,3.9" etc. + m = re.match(r"\s*([0-9]+(?:\.[0-9]+)?)\s*[-,:]?\s*(.+)$", line) + if m: + score = float(m.group(1)) + filename = m.group(2) + else: + parts = re.split(r"[,\t]", line) + if len(parts) >= 2: + try: + score = float(parts[1]) + filename = parts[0] + except Exception: + continue + else: + continue + key = canonical_key(filename) + scores[key] = score + return scores + + +def read_human_labels(csv_path: Path) -> Dict[str, str]: + labels: Dict[str, str] = {} + with csv_path.open("r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + filename = (row.get("filename") or row.get("file") or "").strip() + label = (row.get("label") or row.get("Label") or "").strip() + if not filename: + continue + key = canonical_key(filename) + if label: + labels[key] = label + return labels + + +def levenshtein_distance(a: str, b: str) -> int: + if a == b: + return 0 + if len(a) == 0: + return len(b) + if len(b) == 0: + return len(a) + previous_row = list(range(len(b) + 1)) + for i, ca in enumerate(a, start=1): + current_row = [i] + for j, cb in enumerate(b, start=1): + insertions = previous_row[j] + 1 + deletions = current_row[j - 1] + 1 + substitutions = previous_row[j - 1] + (0 if ca == cb else 1) + current_row.append(min(insertions, deletions, substitutions)) + previous_row = current_row + return previous_row[-1] + + +def normalized_similarity(pred: str, gt: str) -> float: + pred = pred or "" + gt = gt or "" + max_len = max(len(pred), len(gt)) + if max_len == 0: + return 1.0 + dist = levenshtein_distance(pred, gt) + sim = 1.0 - (dist / max_len) + if sim < 0.0: + return 0.0 + if sim > 1.0: + return 1.0 + return sim + + +def compute_anls_for_record(record: Dict) -> tuple[float, int]: + fields = record.get("fields") or [] + if not isinstance(fields, list) or len(fields) == 0: + return 0.0, 0 + sims: List[float] = [] + for f in fields: + pred = str(f.get("pred", "")) + gt = str(f.get("gt", "")) + sims.append(normalized_similarity(pred, gt)) + anls = float(sum(sims) / len(sims)) if sims else 0.0 + return anls, len(sims) + + +def load_per_image_anls(input_json: Path) -> pd.DataFrame: + # Prefer existing per_image_anls.csv, otherwise compute quickly + per_image_csv = input_json.parent / "per_image_anls.csv" + if per_image_csv.exists(): + df = pd.read_csv(per_image_csv) + return df + # Fallback: compute minimal ANLS like in the other script + with input_json.open("r", encoding="utf-8") as f: + data = json.load(f) + rows = [] + for rec in data: + anls, num_fields = compute_anls_for_record(rec) + rows.append({ + "image": rec.get("image"), + "anls": anls, + "hallucination_score": 1.0 - anls, + "num_fields": int(num_fields), + }) + return pd.DataFrame(rows) + + +def main() -> None: + p = argparse.ArgumentParser(description="Compare hallucination across raw/DeQA/Human pipelines over methods") + p.add_argument("inputs", nargs="+", help="per_sample_eval.json files for each method") + p.add_argument("--deqa_txt", required=True, help="Path to DeQA scores txt (e.g., cni.txt)") + p.add_argument("--human_csv", required=True, help="Path to human labels CSV") + p.add_argument("--deqa_threshold", type=float, default=2.6, help="DeQA threshold (>=)") + args = p.parse_args() + + # Load filters + deqa_scores = read_deqa_scores(Path(args.deqa_txt)) + human_labels = read_human_labels(Path(args.human_csv)) + + # Aggregate per method + method_to_df: Dict[str, pd.DataFrame] = {} + for ip in args.inputs: + path = Path(ip) + df = load_per_image_anls(path) + df["method"] = path.parent.name + df["image_key"] = df["image"].apply(canonical_key) + method_to_df[path.parent.name] = df + + # Compute means per pipeline (fair comparison: set excluded images to hallucination=0) + records = [] + for method, df in method_to_df.items(): + raw_mean = float(df["hallucination_score"].mean()) if len(df) else float("nan") + + # DeQA filter: mark DeQA < threshold as hallucination=0, keep all images + df_deqa = df.copy() + mask_deqa = df_deqa["image_key"].map(lambda k: deqa_scores.get(k, None)) + # Set hallucination=0 for images with DeQA < threshold (or missing DeQA) + df_deqa.loc[mask_deqa.isna() | (mask_deqa < args.deqa_threshold), "hallucination_score"] = 0.0 + deqa_mean = float(df_deqa["hallucination_score"].mean()) if len(df_deqa) else float("nan") + + # Human filter: mark Low labels as hallucination=0, keep all images + df_human = df.copy() + mask_human = df_human["image_key"].map(lambda k: human_labels.get(k, "").lower()) + # Set hallucination=0 for images labeled Low (or missing label) + df_human.loc[mask_human != "high", "hallucination_score"] = 0.0 + human_mean = float(df_human["hallucination_score"].mean()) if len(df_human) else float("nan") + + records.append({ + "method": method, + "raw_mean": raw_mean, + "deqa_mean": deqa_mean, + "human_mean": human_mean, + "raw_count": int(len(df)), + "deqa_count": int(len(df_deqa)), # Now equal to raw_count + "human_count": int(len(df_human)), # Now equal to raw_count + }) + + outdir = Path(args.inputs[0]).parent.parent / "combined_anls" / "pipeline" + outdir.mkdir(parents=True, exist_ok=True) + out_csv = outdir / "pipeline_means.csv" + means_df = pd.DataFrame(records).sort_values("method") + means_df.to_csv(out_csv, index=False) + + # 3-line comparison plot over methods (narrower with score annotations) + x = range(len(means_df)) + plt.figure(figsize=(7, 5)) + + # Plot lines and add score annotations + raw_vals = means_df["raw_mean"].values + deqa_vals = means_df["deqa_mean"].values + human_vals = means_df["human_mean"].values + + plt.plot(x, raw_vals, marker="o", label="Raw", linewidth=2, markersize=6) + plt.plot(x, deqa_vals, marker="s", label=f"DeQA >= {args.deqa_threshold}", linewidth=2, markersize=6) + plt.plot(x, human_vals, marker="^", label="Human High", linewidth=2, markersize=6) + + # Annotate each point with its score + for i, (r, d, h) in enumerate(zip(raw_vals, deqa_vals, human_vals)): + plt.annotate(f"{r:.3f}", (i, r), textcoords="offset points", xytext=(0,8), ha='center', fontsize=8) + plt.annotate(f"{d:.3f}", (i, d), textcoords="offset points", xytext=(0,8), ha='center', fontsize=8) + plt.annotate(f"{h:.3f}", (i, h), textcoords="offset points", xytext=(0,8), ha='center', fontsize=8) + + plt.xticks(list(x), means_df["method"].tolist(), rotation=25, ha="right") + plt.ylabel("Mean hallucination (1 - ANLS)") + plt.title("Pipeline comparison over preprocessing methods") + plt.grid(axis="y", linestyle="--", alpha=0.3) + plt.legend() + plt.tight_layout() + out_png = outdir / "pipeline_comparison.png" + plt.savefig(out_png, dpi=160) + plt.close() + + print(f"Saved: {out_csv}") + print(f"Saved: {out_png}") + + +if __name__ == "__main__": + main() + +