{ "current_full_core": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", "input": "eval/irish_core_pii_v1.jsonl", "loader_type": "standard", "device": "cpu", "examples": 37, "batch_size": 8, "min_score": 0.35, "ppsn_min_score": 0.5, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "ACCOUNT_NUMBER", "BANK_ROUTING_NUMBER", "CREDIT_DEBIT_CARD", "EMAIL", "FIRST_NAME", "LAST_NAME", "PASSPORT_NUMBER", "PHONE_NUMBER", "POSTCODE", "PPSN", "SWIFT_BIC" ], "elapsed_seconds": 0.6340816880110651, "examples_per_second": 58.35210304851814, "overall": { "precision": 0.9259259259259259, "recall": 0.9868421052631579, "f1": 0.9554140127388535, "tp": 75, "fp": 6, "fn": 1 }, "by_label": { "ACCOUNT_NUMBER": { "precision": 0.6, "recall": 1.0, "f1": 0.7499999999999999, "tp": 3, "fp": 2, "fn": 0 }, "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "fn": 0 }, "CREDIT_DEBIT_CARD": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 }, "EMAIL": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "FIRST_NAME": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "LAST_NAME": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 2, "fp": 1, "fn": 0 }, "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 12, "fp": 0, "fn": 0 }, "POSTCODE": { "precision": 0.75, "recall": 0.75, "f1": 0.75, "tp": 3, "fp": 1, "fn": 1 }, "PPSN": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "SWIFT_BIC": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 } } }, "current_full_edge": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", "input": "eval/irish_ppsn_phone_edge_v1.jsonl", "loader_type": "standard", "device": "cpu", "examples": 22, "batch_size": 8, "min_score": 0.35, "ppsn_min_score": 0.5, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PHONE_NUMBER", "PPSN" ], "elapsed_seconds": 0.34832625201670453, "examples_per_second": 63.15917870854292, "overall": { "precision": 0.9047619047619048, "recall": 1.0, "f1": 0.9500000000000001, "tp": 19, "fp": 2, "fn": 0 }, "by_label": { "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 13, "fp": 0, "fn": 0 }, "PPSN": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 } } }, "current_full_numeric": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", "input": "eval/irish_numeric_qafix_v2.jsonl", "loader_type": "standard", "device": "cpu", "examples": 19, "batch_size": 8, "min_score": 0.35, "ppsn_min_score": 0.5, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "BANK_ROUTING_NUMBER", "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.29271358298137784, "examples_per_second": 64.90986788682356, "overall": { "precision": 0.8666666666666667, "recall": 0.9285714285714286, "f1": 0.896551724137931, "tp": 13, "fp": 2, "fn": 1 }, "by_label": { "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 5, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091, "tp": 5, "fp": 1, "fn": 0 }, "PHONE_NUMBER": { "precision": 0.75, "recall": 0.75, "f1": 0.75, "tp": 3, "fp": 1, "fn": 1 } } }, "current_full_gap": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", "input": "eval/irish_core_gap_feedback_v1.jsonl", "loader_type": "standard", "device": "cpu", "examples": 14, "batch_size": 8, "min_score": 0.35, "ppsn_min_score": 0.5, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "BANK_ROUTING_NUMBER", "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.2173796920105815, "examples_per_second": 64.40344022255086, "overall": { "precision": 0.8333333333333334, "recall": 0.9090909090909091, "f1": 0.8695652173913043, "tp": 10, "fp": 2, "fn": 1 }, "by_label": { "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 4, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 0.8, "recall": 1.0, "f1": 0.888888888888889, "tp": 4, "fp": 1, "fn": 0 }, "PHONE_NUMBER": { "precision": 0.6666666666666666, "recall": 0.6666666666666666, "f1": 0.6666666666666666, "tp": 2, "fp": 1, "fn": 1 } } }, "current_full_user": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", "input": "eval/user_raw_regression_cases_v1.jsonl", "loader_type": "standard", "device": "cpu", "examples": 7, "batch_size": 8, "min_score": 0.35, "ppsn_min_score": 0.5, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PPSN" ], "elapsed_seconds": 0.11388953996356577, "examples_per_second": 61.46306326497903, "overall": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 3, "fp": 1, "fn": 0 }, "by_label": { "PHONE_NUMBER": { "precision": 0.0, "recall": 0.0, "f1": 0.0, "tp": 0, "fp": 1, "fn": 0 }, "PPSN": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 3, "fp": 0, "fn": 0 } } }, "current_full_remaining": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", "input": "eval/irish_remaining_gaps_v2.jsonl", "loader_type": "standard", "device": "cpu", "examples": 6, "batch_size": 8, "min_score": 0.35, "ppsn_min_score": 0.5, "iou_threshold": 1.0, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.10174437699606642, "examples_per_second": 58.97131789633906, "overall": { "precision": 0.4, "recall": 0.4, "f1": 0.4000000000000001, "tp": 2, "fp": 3, "fn": 3 }, "by_label": { "PASSPORT_NUMBER": { "precision": 0.0, "recall": 0.0, "f1": 0.0, "tp": 0, "fp": 2, "fn": 2 }, "PHONE_NUMBER": { "precision": 0.6666666666666666, "recall": 0.6666666666666666, "f1": 0.6666666666666666, "tp": 2, "fp": 1, "fn": 1 } } }, "current_full_multilingual": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", "input": "eval/multilingual_ppsn_v1_all.jsonl", "loader_type": "standard", "device": "cpu", "examples": 168, "batch_size": 16, "ppsn_decoder": "word_aligned", "elapsed_seconds": 1.310564298008103, "examples_per_second": 128.18905585581678, "label_filter": "PPSN", "overall": { "precision": 0.672, "recall": 1.0, "f1": 0.8038277511961722, "tp": 84, "fp": 41, "fn": 0 }, "by_domain": { "citizen_chat": { "precision": 0.6511627906976745, "recall": 1.0, "f1": 0.7887323943661972, "tp": 28, "fp": 15, "fn": 0 }, "gov_data": { "precision": 0.7, "recall": 1.0, "f1": 0.8235294117647058, "tp": 28, "fp": 12, "fn": 0 }, "hse_medical": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 28, "fp": 14, "fn": 0 } }, "by_language": { "de": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "en": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "es": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "fr": { "precision": 0.8571428571428571, "recall": 1.0, "f1": 0.923076923076923, "tp": 6, "fp": 1, "fn": 0 }, "ga": { "precision": 0.5, "recall": 1.0, "f1": 0.6666666666666666, "tp": 6, "fp": 6, "fn": 0 }, "it": { "precision": 0.6, "recall": 1.0, "f1": 0.7499999999999999, "tp": 6, "fp": 4, "fn": 0 }, "ja": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "nl": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "pl": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "pt": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "ro": { "precision": 0.6, "recall": 1.0, "f1": 0.7499999999999999, "tp": 6, "fp": 4, "fn": 0 }, "ru": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "uk": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "zh": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 } } }, "candidate_full_core": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", "input": "eval/irish_core_pii_v1.jsonl", "loader_type": "standard", "device": "cpu", "examples": 37, "batch_size": 8, "min_score": 0.4, "ppsn_min_score": 0.55, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "ACCOUNT_NUMBER", "BANK_ROUTING_NUMBER", "CREDIT_DEBIT_CARD", "EMAIL", "FIRST_NAME", "LAST_NAME", "PASSPORT_NUMBER", "PHONE_NUMBER", "POSTCODE", "PPSN", "SWIFT_BIC" ], "elapsed_seconds": 0.5703115339856595, "examples_per_second": 64.87682221929316, "overall": { "precision": 0.9620253164556962, "recall": 1.0, "f1": 0.9806451612903226, "tp": 76, "fp": 3, "fn": 0 }, "by_label": { "ACCOUNT_NUMBER": { "precision": 0.6, "recall": 1.0, "f1": 0.7499999999999999, "tp": 3, "fp": 2, "fn": 0 }, "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "fn": 0 }, "CREDIT_DEBIT_CARD": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 }, "EMAIL": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "FIRST_NAME": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "LAST_NAME": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 }, "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 12, "fp": 0, "fn": 0 }, "POSTCODE": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 4, "fp": 0, "fn": 0 }, "PPSN": { "precision": 0.8571428571428571, "recall": 1.0, "f1": 0.923076923076923, "tp": 6, "fp": 1, "fn": 0 }, "SWIFT_BIC": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 } } }, "candidate_full_edge": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", "input": "eval/irish_ppsn_phone_edge_v1.jsonl", "loader_type": "standard", "device": "cpu", "examples": 22, "batch_size": 8, "min_score": 0.4, "ppsn_min_score": 0.55, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PHONE_NUMBER", "PPSN" ], "elapsed_seconds": 0.35221877897856757, "examples_per_second": 62.46117842949735, "overall": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "by_label": { "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 13, "fp": 0, "fn": 0 }, "PPSN": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 } } }, "candidate_full_numeric": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", "input": "eval/irish_numeric_qafix_v2.jsonl", "loader_type": "standard", "device": "cpu", "examples": 19, "batch_size": 8, "min_score": 0.4, "ppsn_min_score": 0.55, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "BANK_ROUTING_NUMBER", "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.2983117959811352, "examples_per_second": 63.69174888813827, "overall": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 14, "fp": 2, "fn": 0 }, "by_label": { "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 5, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091, "tp": 5, "fp": 1, "fn": 0 }, "PHONE_NUMBER": { "precision": 0.8, "recall": 1.0, "f1": 0.888888888888889, "tp": 4, "fp": 1, "fn": 0 } } }, "candidate_full_gap": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", "input": "eval/irish_core_gap_feedback_v1.jsonl", "loader_type": "standard", "device": "cpu", "examples": 14, "batch_size": 8, "min_score": 0.4, "ppsn_min_score": 0.55, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "BANK_ROUTING_NUMBER", "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.229215633997228, "examples_per_second": 61.07785824142043, "overall": { "precision": 0.8461538461538461, "recall": 1.0, "f1": 0.9166666666666666, "tp": 11, "fp": 2, "fn": 0 }, "by_label": { "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 4, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 0.8, "recall": 1.0, "f1": 0.888888888888889, "tp": 4, "fp": 1, "fn": 0 }, "PHONE_NUMBER": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 3, "fp": 1, "fn": 0 } } }, "candidate_full_user": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", "input": "eval/user_raw_regression_cases_v1.jsonl", "loader_type": "standard", "device": "cpu", "examples": 7, "batch_size": 8, "min_score": 0.4, "ppsn_min_score": 0.55, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PPSN" ], "elapsed_seconds": 0.11162209301255643, "examples_per_second": 62.711599568488346, "overall": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 3, "fp": 0, "fn": 0 }, "by_label": { "PPSN": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 3, "fp": 0, "fn": 0 } } }, "candidate_full_remaining": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", "input": "eval/irish_remaining_gaps_v2.jsonl", "loader_type": "standard", "device": "cpu", "examples": 6, "batch_size": 8, "min_score": 0.4, "ppsn_min_score": 0.55, "iou_threshold": 1.0, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.09866602800320834, "examples_per_second": 60.8112044381162, "overall": { "precision": 0.5, "recall": 0.4, "f1": 0.4444444444444445, "tp": 2, "fp": 2, "fn": 3 }, "by_label": { "PASSPORT_NUMBER": { "precision": 0.0, "recall": 0.0, "f1": 0.0, "tp": 0, "fp": 1, "fn": 2 }, "PHONE_NUMBER": { "precision": 0.6666666666666666, "recall": 0.6666666666666666, "f1": 0.6666666666666666, "tp": 2, "fp": 1, "fn": 1 } } }, "candidate_full_multilingual": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", "input": "eval/multilingual_ppsn_v1_all.jsonl", "loader_type": "standard", "device": "cpu", "examples": 168, "batch_size": 16, "ppsn_decoder": "word_aligned", "elapsed_seconds": 1.3280671570100822, "examples_per_second": 126.49962700547727, "label_filter": "PPSN", "overall": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 84, "fp": 12, "fn": 0 }, "by_domain": { "citizen_chat": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 28, "fp": 4, "fn": 0 }, "gov_data": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 28, "fp": 4, "fn": 0 }, "hse_medical": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 28, "fp": 4, "fn": 0 } }, "by_language": { "de": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "en": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "es": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "fr": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "ga": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "it": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "ja": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "nl": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "pl": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "pt": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "ro": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "ru": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "uk": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "zh": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 } } }, "current_q8_core": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", "input": "eval/irish_core_pii_v1.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 37, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "ACCOUNT_NUMBER", "BANK_ROUTING_NUMBER", "CREDIT_DEBIT_CARD", "EMAIL", "FIRST_NAME", "LAST_NAME", "PASSPORT_NUMBER", "PHONE_NUMBER", "POSTCODE", "PPSN", "SWIFT_BIC" ], "elapsed_seconds": 0.8229939279844984, "examples_per_second": 44.957804355388774, "overall": { "precision": 0.9493670886075949, "recall": 0.9868421052631579, "f1": 0.967741935483871, "tp": 75, "fp": 4, "fn": 1 }, "by_label": { "ACCOUNT_NUMBER": { "precision": 0.6, "recall": 1.0, "f1": 0.7499999999999999, "tp": 3, "fp": 2, "fn": 0 }, "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "fn": 0 }, "CREDIT_DEBIT_CARD": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 }, "EMAIL": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "FIRST_NAME": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "LAST_NAME": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 }, "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 12, "fp": 0, "fn": 0 }, "POSTCODE": { "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571, "tp": 3, "fp": 0, "fn": 1 }, "PPSN": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "SWIFT_BIC": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 } } }, "current_q8_edge": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", "input": "eval/irish_ppsn_phone_edge_v1.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 22, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PHONE_NUMBER", "PPSN" ], "elapsed_seconds": 0.4826828690129332, "examples_per_second": 45.57858049735451, "overall": { "precision": 0.9047619047619048, "recall": 1.0, "f1": 0.9500000000000001, "tp": 19, "fp": 2, "fn": 0 }, "by_label": { "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 13, "fp": 0, "fn": 0 }, "PPSN": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 } } }, "current_q8_numeric": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", "input": "eval/irish_numeric_qafix_v2.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 19, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "BANK_ROUTING_NUMBER", "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.48927704500965774, "examples_per_second": 38.8328048368281, "overall": { "precision": 0.8125, "recall": 0.9285714285714286, "f1": 0.8666666666666666, "tp": 13, "fp": 3, "fn": 1 }, "by_label": { "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 5, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 0.7142857142857143, "recall": 1.0, "f1": 0.8333333333333333, "tp": 5, "fp": 2, "fn": 0 }, "PHONE_NUMBER": { "precision": 0.75, "recall": 0.75, "f1": 0.75, "tp": 3, "fp": 1, "fn": 1 } } }, "current_q8_gap": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", "input": "eval/irish_core_gap_feedback_v1.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 14, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "BANK_ROUTING_NUMBER", "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.25282940798206255, "examples_per_second": 55.37330531183009, "overall": { "precision": 0.8333333333333334, "recall": 0.9090909090909091, "f1": 0.8695652173913043, "tp": 10, "fp": 2, "fn": 1 }, "by_label": { "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 4, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 0.8, "recall": 1.0, "f1": 0.888888888888889, "tp": 4, "fp": 1, "fn": 0 }, "PHONE_NUMBER": { "precision": 0.6666666666666666, "recall": 0.6666666666666666, "f1": 0.6666666666666666, "tp": 2, "fp": 1, "fn": 1 } } }, "current_q8_user": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", "input": "eval/user_raw_regression_cases_v1.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 7, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PPSN" ], "elapsed_seconds": 0.15658493799855933, "examples_per_second": 44.70417199427192, "overall": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 3, "fp": 1, "fn": 0 }, "by_label": { "PHONE_NUMBER": { "precision": 0.0, "recall": 0.0, "f1": 0.0, "tp": 0, "fp": 1, "fn": 0 }, "PPSN": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 3, "fp": 0, "fn": 0 } } }, "current_q8_remaining": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", "input": "eval/irish_remaining_gaps_v2.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 6, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 1.0, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.1308825800078921, "examples_per_second": 45.84261709723483, "overall": { "precision": 0.6, "recall": 0.6, "f1": 0.6, "tp": 3, "fp": 2, "fn": 2 }, "by_label": { "PASSPORT_NUMBER": { "precision": 0.0, "recall": 0.0, "f1": 0.0, "tp": 0, "fp": 2, "fn": 2 }, "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 3, "fp": 0, "fn": 0 } } }, "current_q8_multilingual": { "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", "input": "eval/multilingual_ppsn_v1_all.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 168, "batch_size": 16, "ppsn_decoder": "word_aligned", "elapsed_seconds": 0.5646560749737546, "examples_per_second": 297.5262419833324, "label_filter": "PPSN", "overall": { "precision": 0.6774193548387096, "recall": 1.0, "f1": 0.8076923076923077, "tp": 84, "fp": 40, "fn": 0 }, "by_domain": { "citizen_chat": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 28, "fp": 14, "fn": 0 }, "gov_data": { "precision": 0.6829268292682927, "recall": 1.0, "f1": 0.8115942028985507, "tp": 28, "fp": 13, "fn": 0 }, "hse_medical": { "precision": 0.6829268292682927, "recall": 1.0, "f1": 0.8115942028985507, "tp": 28, "fp": 13, "fn": 0 } }, "by_language": { "de": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "en": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "es": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "fr": { "precision": 0.8571428571428571, "recall": 1.0, "f1": 0.923076923076923, "tp": 6, "fp": 1, "fn": 0 }, "ga": { "precision": 0.5454545454545454, "recall": 1.0, "f1": 0.7058823529411764, "tp": 6, "fp": 5, "fn": 0 }, "it": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "ja": { "precision": 0.6, "recall": 1.0, "f1": 0.7499999999999999, "tp": 6, "fp": 4, "fn": 0 }, "nl": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "pl": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "pt": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "ro": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "ru": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "uk": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "zh": { "precision": 0.6, "recall": 1.0, "f1": 0.7499999999999999, "tp": 6, "fp": 4, "fn": 0 } }, "onnx_file": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8/model_quantized.onnx" }, "candidate_q8_core": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", "input": "eval/irish_core_pii_v1.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 37, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "ACCOUNT_NUMBER", "BANK_ROUTING_NUMBER", "CREDIT_DEBIT_CARD", "EMAIL", "FIRST_NAME", "LAST_NAME", "PASSPORT_NUMBER", "PHONE_NUMBER", "POSTCODE", "PPSN", "SWIFT_BIC" ], "elapsed_seconds": 0.8233243200229481, "examples_per_second": 44.93976322595295, "overall": { "precision": 0.9493670886075949, "recall": 0.9868421052631579, "f1": 0.967741935483871, "tp": 75, "fp": 4, "fn": 1 }, "by_label": { "ACCOUNT_NUMBER": { "precision": 0.6, "recall": 1.0, "f1": 0.7499999999999999, "tp": 3, "fp": 2, "fn": 0 }, "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "fn": 0 }, "CREDIT_DEBIT_CARD": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 }, "EMAIL": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "FIRST_NAME": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "LAST_NAME": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 }, "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 12, "fp": 0, "fn": 0 }, "POSTCODE": { "precision": 0.75, "recall": 0.75, "f1": 0.75, "tp": 3, "fp": 1, "fn": 1 }, "PPSN": { "precision": 0.8571428571428571, "recall": 1.0, "f1": 0.923076923076923, "tp": 6, "fp": 1, "fn": 0 }, "SWIFT_BIC": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 2, "fp": 0, "fn": 0 } } }, "candidate_q8_edge": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", "input": "eval/irish_ppsn_phone_edge_v1.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 22, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PHONE_NUMBER", "PPSN" ], "elapsed_seconds": 0.5273098170291632, "examples_per_second": 41.72120315139757, "overall": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 19, "fp": 0, "fn": 0 }, "by_label": { "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 13, "fp": 0, "fn": 0 }, "PPSN": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 } } }, "candidate_q8_numeric": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", "input": "eval/irish_numeric_qafix_v2.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 19, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "BANK_ROUTING_NUMBER", "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.3350751459947787, "examples_per_second": 56.70369834083745, "overall": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 14, "fp": 2, "fn": 0 }, "by_label": { "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 5, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091, "tp": 5, "fp": 1, "fn": 0 }, "PHONE_NUMBER": { "precision": 0.8, "recall": 1.0, "f1": 0.888888888888889, "tp": 4, "fp": 1, "fn": 0 } } }, "candidate_q8_gap": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", "input": "eval/irish_core_gap_feedback_v1.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 14, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "BANK_ROUTING_NUMBER", "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.1985326989670284, "examples_per_second": 70.51735090915713, "overall": { "precision": 0.8461538461538461, "recall": 1.0, "f1": 0.9166666666666666, "tp": 11, "fp": 2, "fn": 0 }, "by_label": { "BANK_ROUTING_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 4, "fp": 0, "fn": 0 }, "PASSPORT_NUMBER": { "precision": 0.8, "recall": 1.0, "f1": 0.888888888888889, "tp": 4, "fp": 1, "fn": 0 }, "PHONE_NUMBER": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 3, "fp": 1, "fn": 0 } } }, "candidate_q8_user": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", "input": "eval/user_raw_regression_cases_v1.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 7, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 0.5, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PPSN" ], "elapsed_seconds": 0.13422382500721142, "examples_per_second": 52.1516951228585, "overall": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 3, "fp": 0, "fn": 0 }, "by_label": { "PPSN": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 3, "fp": 0, "fn": 0 } } }, "candidate_q8_remaining": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", "input": "eval/irish_remaining_gaps_v2.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 6, "batch_size": 8, "min_score": 0.45, "ppsn_min_score": 0.6, "iou_threshold": 1.0, "ppsn_decoder": "word_aligned", "labels_evaluated": [ "PASSPORT_NUMBER", "PHONE_NUMBER" ], "elapsed_seconds": 0.142813170037698, "examples_per_second": 42.01293198950906, "overall": { "precision": 0.75, "recall": 0.6, "f1": 0.6666666666666665, "tp": 3, "fp": 1, "fn": 2 }, "by_label": { "PASSPORT_NUMBER": { "precision": 0.0, "recall": 0.0, "f1": 0.0, "tp": 0, "fp": 1, "fn": 2 }, "PHONE_NUMBER": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 3, "fp": 0, "fn": 0 } } }, "candidate_q8_multilingual": { "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", "input": "eval/multilingual_ppsn_v1_all.jsonl", "loader_type": "onnx", "device": "cpu", "examples": 168, "batch_size": 16, "ppsn_decoder": "word_aligned", "elapsed_seconds": 0.8803652579663321, "examples_per_second": 190.8298839371338, "label_filter": "PPSN", "overall": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 84, "fp": 12, "fn": 0 }, "by_domain": { "citizen_chat": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 28, "fp": 4, "fn": 0 }, "gov_data": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 28, "fp": 4, "fn": 0 }, "hse_medical": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "tp": 28, "fp": 4, "fn": 0 } }, "by_language": { "de": { "precision": 0.8571428571428571, "recall": 1.0, "f1": 0.923076923076923, "tp": 6, "fp": 1, "fn": 0 }, "en": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "es": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "fr": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "ga": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "it": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "ja": { "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571, "tp": 6, "fp": 2, "fn": 0 }, "nl": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "pl": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "pt": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "ro": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 6, "fp": 0, "fn": 0 }, "ru": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "uk": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 }, "zh": { "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8, "tp": 6, "fp": 3, "fn": 0 } }, "onnx_file": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8/model_quantized.onnx" }, "q8_recipe_compare": [ { "variant": "current_q8", "core_f1": 0.9615384615384615, "core_eps": 119.06137431000792, "edge_f1": 0.9500000000000001, "edge_eps": 69.96690039628702, "numeric_f1": 0.8387096774193549, "numeric_eps": 28.129100872864512, "gap_f1": 0.8333333333333333, "gap_eps": 41.070526382683134, "multilingual_f1": 0.7887323943661972, "multilingual_eps": 86.94930904250197, "remaining_iou1_f1": 0.6, "remaining_iou1_eps": 117.17196092182134 }, { "variant": "q8_mg", "core_f1": 0.9615384615384615, "core_eps": 62.54552479459656, "edge_f1": 0.9500000000000001, "edge_eps": 100.26464304046029, "numeric_f1": 0.8387096774193549, "numeric_eps": 47.47899731078072, "gap_f1": 0.8333333333333333, "gap_eps": 71.39182685847751, "multilingual_f1": 0.7368421052631579, "multilingual_eps": 162.2228381957003, "remaining_iou1_f1": 0.6, "remaining_iou1_eps": 66.84971455324192 }, { "variant": "q8_mg_reduce", "core_f1": 0.9554140127388535, "core_eps": 111.07958753347033, "edge_f1": 0.9500000000000001, "edge_eps": 50.58773730012114, "numeric_f1": 0.7878787878787878, "numeric_eps": 42.08249226466411, "gap_f1": 0.7692307692307692, "gap_eps": 112.74792550347136, "multilingual_f1": 0.7336244541484717, "multilingual_eps": 105.25341259054518, "remaining_iou1_f1": 0.4000000000000001, "remaining_iou1_eps": 54.19501381319339 } ] }