| { |
| "current_full_core": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_core_pii_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 37, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "ACCOUNT_NUMBER", |
| "BANK_ROUTING_NUMBER", |
| "CREDIT_DEBIT_CARD", |
| "EMAIL", |
| "FIRST_NAME", |
| "LAST_NAME", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER", |
| "POSTCODE", |
| "PPSN", |
| "SWIFT_BIC" |
| ], |
| "elapsed_seconds": 0.6340816880110651, |
| "examples_per_second": 58.35210304851814, |
| "overall": { |
| "precision": 0.9259259259259259, |
| "recall": 0.9868421052631579, |
| "f1": 0.9554140127388535, |
| "tp": 75, |
| "fp": 6, |
| "fn": 1 |
| }, |
| "by_label": { |
| "ACCOUNT_NUMBER": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 3, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 1, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "CREDIT_DEBIT_CARD": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "EMAIL": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "FIRST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "LAST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 2, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 12, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "POSTCODE": { |
| "precision": 0.75, |
| "recall": 0.75, |
| "f1": 0.75, |
| "tp": 3, |
| "fp": 1, |
| "fn": 1 |
| }, |
| "PPSN": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "SWIFT_BIC": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "current_full_edge": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_ppsn_phone_edge_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 22, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PHONE_NUMBER", |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.34832625201670453, |
| "examples_per_second": 63.15917870854292, |
| "overall": { |
| "precision": 0.9047619047619048, |
| "recall": 1.0, |
| "f1": 0.9500000000000001, |
| "tp": 19, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 13, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| } |
| } |
| }, |
| "current_full_numeric": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_numeric_qafix_v2.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 19, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.29271358298137784, |
| "examples_per_second": 64.90986788682356, |
| "overall": { |
| "precision": 0.8666666666666667, |
| "recall": 0.9285714285714286, |
| "f1": 0.896551724137931, |
| "tp": 13, |
| "fp": 2, |
| "fn": 1 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 5, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8333333333333334, |
| "recall": 1.0, |
| "f1": 0.9090909090909091, |
| "tp": 5, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.75, |
| "recall": 0.75, |
| "f1": 0.75, |
| "tp": 3, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "current_full_gap": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_core_gap_feedback_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 14, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.2173796920105815, |
| "examples_per_second": 64.40344022255086, |
| "overall": { |
| "precision": 0.8333333333333334, |
| "recall": 0.9090909090909091, |
| "f1": 0.8695652173913043, |
| "tp": 10, |
| "fp": 2, |
| "fn": 1 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "tp": 2, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "current_full_user": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/user_raw_regression_cases_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 7, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.11388953996356577, |
| "examples_per_second": 61.46306326497903, |
| "overall": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "current_full_remaining": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_remaining_gaps_v2.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 6, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 1.0, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.10174437699606642, |
| "examples_per_second": 58.97131789633906, |
| "overall": { |
| "precision": 0.4, |
| "recall": 0.4, |
| "f1": 0.4000000000000001, |
| "tp": 2, |
| "fp": 3, |
| "fn": 3 |
| }, |
| "by_label": { |
| "PASSPORT_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 2, |
| "fn": 2 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "tp": 2, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "current_full_multilingual": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/multilingual_ppsn_v1_all.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 168, |
| "batch_size": 16, |
| "ppsn_decoder": "word_aligned", |
| "elapsed_seconds": 1.310564298008103, |
| "examples_per_second": 128.18905585581678, |
| "label_filter": "PPSN", |
| "overall": { |
| "precision": 0.672, |
| "recall": 1.0, |
| "f1": 0.8038277511961722, |
| "tp": 84, |
| "fp": 41, |
| "fn": 0 |
| }, |
| "by_domain": { |
| "citizen_chat": { |
| "precision": 0.6511627906976745, |
| "recall": 1.0, |
| "f1": 0.7887323943661972, |
| "tp": 28, |
| "fp": 15, |
| "fn": 0 |
| }, |
| "gov_data": { |
| "precision": 0.7, |
| "recall": 1.0, |
| "f1": 0.8235294117647058, |
| "tp": 28, |
| "fp": 12, |
| "fn": 0 |
| }, |
| "hse_medical": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 28, |
| "fp": 14, |
| "fn": 0 |
| } |
| }, |
| "by_language": { |
| "de": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "en": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "es": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "fr": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "ga": { |
| "precision": 0.5, |
| "recall": 1.0, |
| "f1": 0.6666666666666666, |
| "tp": 6, |
| "fp": 6, |
| "fn": 0 |
| }, |
| "it": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 6, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "ja": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "nl": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "pl": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "pt": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "ro": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 6, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "ru": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "uk": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "zh": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_full_core": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_core_pii_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 37, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "ACCOUNT_NUMBER", |
| "BANK_ROUTING_NUMBER", |
| "CREDIT_DEBIT_CARD", |
| "EMAIL", |
| "FIRST_NAME", |
| "LAST_NAME", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER", |
| "POSTCODE", |
| "PPSN", |
| "SWIFT_BIC" |
| ], |
| "elapsed_seconds": 0.5703115339856595, |
| "examples_per_second": 64.87682221929316, |
| "overall": { |
| "precision": 0.9620253164556962, |
| "recall": 1.0, |
| "f1": 0.9806451612903226, |
| "tp": 76, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "by_label": { |
| "ACCOUNT_NUMBER": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 3, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 1, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "CREDIT_DEBIT_CARD": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "EMAIL": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "FIRST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "LAST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 12, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "POSTCODE": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "SWIFT_BIC": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_full_edge": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_ppsn_phone_edge_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 22, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PHONE_NUMBER", |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.35221877897856757, |
| "examples_per_second": 62.46117842949735, |
| "overall": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 13, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_full_numeric": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_numeric_qafix_v2.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 19, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.2983117959811352, |
| "examples_per_second": 63.69174888813827, |
| "overall": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 14, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 5, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8333333333333334, |
| "recall": 1.0, |
| "f1": 0.9090909090909091, |
| "tp": 5, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_full_gap": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_core_gap_feedback_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 14, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.229215633997228, |
| "examples_per_second": 61.07785824142043, |
| "overall": { |
| "precision": 0.8461538461538461, |
| "recall": 1.0, |
| "f1": 0.9166666666666666, |
| "tp": 11, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 1, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_full_user": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/user_raw_regression_cases_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 7, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.11162209301255643, |
| "examples_per_second": 62.711599568488346, |
| "overall": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_full_remaining": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_remaining_gaps_v2.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 6, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 1.0, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.09866602800320834, |
| "examples_per_second": 60.8112044381162, |
| "overall": { |
| "precision": 0.5, |
| "recall": 0.4, |
| "f1": 0.4444444444444445, |
| "tp": 2, |
| "fp": 2, |
| "fn": 3 |
| }, |
| "by_label": { |
| "PASSPORT_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 1, |
| "fn": 2 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "tp": 2, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "candidate_full_multilingual": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/multilingual_ppsn_v1_all.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 168, |
| "batch_size": 16, |
| "ppsn_decoder": "word_aligned", |
| "elapsed_seconds": 1.3280671570100822, |
| "examples_per_second": 126.49962700547727, |
| "label_filter": "PPSN", |
| "overall": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 84, |
| "fp": 12, |
| "fn": 0 |
| }, |
| "by_domain": { |
| "citizen_chat": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "gov_data": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "hse_medical": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| } |
| }, |
| "by_language": { |
| "de": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "en": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "es": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "fr": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ga": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "it": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ja": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "nl": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "pl": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "pt": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ro": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ru": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "uk": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "zh": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| } |
| } |
| }, |
| "current_q8_core": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_core_pii_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 37, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "ACCOUNT_NUMBER", |
| "BANK_ROUTING_NUMBER", |
| "CREDIT_DEBIT_CARD", |
| "EMAIL", |
| "FIRST_NAME", |
| "LAST_NAME", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER", |
| "POSTCODE", |
| "PPSN", |
| "SWIFT_BIC" |
| ], |
| "elapsed_seconds": 0.8229939279844984, |
| "examples_per_second": 44.957804355388774, |
| "overall": { |
| "precision": 0.9493670886075949, |
| "recall": 0.9868421052631579, |
| "f1": 0.967741935483871, |
| "tp": 75, |
| "fp": 4, |
| "fn": 1 |
| }, |
| "by_label": { |
| "ACCOUNT_NUMBER": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 3, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 1, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "CREDIT_DEBIT_CARD": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "EMAIL": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "FIRST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "LAST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 12, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "POSTCODE": { |
| "precision": 1.0, |
| "recall": 0.75, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 0, |
| "fn": 1 |
| }, |
| "PPSN": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "SWIFT_BIC": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "current_q8_edge": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_ppsn_phone_edge_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 22, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PHONE_NUMBER", |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.4826828690129332, |
| "examples_per_second": 45.57858049735451, |
| "overall": { |
| "precision": 0.9047619047619048, |
| "recall": 1.0, |
| "f1": 0.9500000000000001, |
| "tp": 19, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 13, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| } |
| } |
| }, |
| "current_q8_numeric": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_numeric_qafix_v2.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 19, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.48927704500965774, |
| "examples_per_second": 38.8328048368281, |
| "overall": { |
| "precision": 0.8125, |
| "recall": 0.9285714285714286, |
| "f1": 0.8666666666666666, |
| "tp": 13, |
| "fp": 3, |
| "fn": 1 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 5, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.7142857142857143, |
| "recall": 1.0, |
| "f1": 0.8333333333333333, |
| "tp": 5, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.75, |
| "recall": 0.75, |
| "f1": 0.75, |
| "tp": 3, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "current_q8_gap": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_core_gap_feedback_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 14, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.25282940798206255, |
| "examples_per_second": 55.37330531183009, |
| "overall": { |
| "precision": 0.8333333333333334, |
| "recall": 0.9090909090909091, |
| "f1": 0.8695652173913043, |
| "tp": 10, |
| "fp": 2, |
| "fn": 1 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "tp": 2, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "current_q8_user": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/user_raw_regression_cases_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 7, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.15658493799855933, |
| "examples_per_second": 44.70417199427192, |
| "overall": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "current_q8_remaining": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_remaining_gaps_v2.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 6, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 1.0, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.1308825800078921, |
| "examples_per_second": 45.84261709723483, |
| "overall": { |
| "precision": 0.6, |
| "recall": 0.6, |
| "f1": 0.6, |
| "tp": 3, |
| "fp": 2, |
| "fn": 2 |
| }, |
| "by_label": { |
| "PASSPORT_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 2, |
| "fn": 2 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "current_q8_multilingual": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/multilingual_ppsn_v1_all.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 168, |
| "batch_size": 16, |
| "ppsn_decoder": "word_aligned", |
| "elapsed_seconds": 0.5646560749737546, |
| "examples_per_second": 297.5262419833324, |
| "label_filter": "PPSN", |
| "overall": { |
| "precision": 0.6774193548387096, |
| "recall": 1.0, |
| "f1": 0.8076923076923077, |
| "tp": 84, |
| "fp": 40, |
| "fn": 0 |
| }, |
| "by_domain": { |
| "citizen_chat": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 28, |
| "fp": 14, |
| "fn": 0 |
| }, |
| "gov_data": { |
| "precision": 0.6829268292682927, |
| "recall": 1.0, |
| "f1": 0.8115942028985507, |
| "tp": 28, |
| "fp": 13, |
| "fn": 0 |
| }, |
| "hse_medical": { |
| "precision": 0.6829268292682927, |
| "recall": 1.0, |
| "f1": 0.8115942028985507, |
| "tp": 28, |
| "fp": 13, |
| "fn": 0 |
| } |
| }, |
| "by_language": { |
| "de": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "en": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "es": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "fr": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "ga": { |
| "precision": 0.5454545454545454, |
| "recall": 1.0, |
| "f1": 0.7058823529411764, |
| "tp": 6, |
| "fp": 5, |
| "fn": 0 |
| }, |
| "it": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "ja": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 6, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "nl": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "pl": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "pt": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "ro": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "ru": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "uk": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "zh": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 6, |
| "fp": 4, |
| "fn": 0 |
| } |
| }, |
| "onnx_file": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8/model_quantized.onnx" |
| }, |
| "candidate_q8_core": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_core_pii_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 37, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "ACCOUNT_NUMBER", |
| "BANK_ROUTING_NUMBER", |
| "CREDIT_DEBIT_CARD", |
| "EMAIL", |
| "FIRST_NAME", |
| "LAST_NAME", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER", |
| "POSTCODE", |
| "PPSN", |
| "SWIFT_BIC" |
| ], |
| "elapsed_seconds": 0.8233243200229481, |
| "examples_per_second": 44.93976322595295, |
| "overall": { |
| "precision": 0.9493670886075949, |
| "recall": 0.9868421052631579, |
| "f1": 0.967741935483871, |
| "tp": 75, |
| "fp": 4, |
| "fn": 1 |
| }, |
| "by_label": { |
| "ACCOUNT_NUMBER": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 3, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 1, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "CREDIT_DEBIT_CARD": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "EMAIL": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "FIRST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "LAST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 12, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "POSTCODE": { |
| "precision": 0.75, |
| "recall": 0.75, |
| "f1": 0.75, |
| "tp": 3, |
| "fp": 1, |
| "fn": 1 |
| }, |
| "PPSN": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "SWIFT_BIC": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_q8_edge": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_ppsn_phone_edge_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 22, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PHONE_NUMBER", |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.5273098170291632, |
| "examples_per_second": 41.72120315139757, |
| "overall": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 13, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_q8_numeric": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_numeric_qafix_v2.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 19, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.3350751459947787, |
| "examples_per_second": 56.70369834083745, |
| "overall": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 14, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 5, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8333333333333334, |
| "recall": 1.0, |
| "f1": 0.9090909090909091, |
| "tp": 5, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_q8_gap": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_core_gap_feedback_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 14, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.1985326989670284, |
| "examples_per_second": 70.51735090915713, |
| "overall": { |
| "precision": 0.8461538461538461, |
| "recall": 1.0, |
| "f1": 0.9166666666666666, |
| "tp": 11, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 1, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_q8_user": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/user_raw_regression_cases_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 7, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.13422382500721142, |
| "examples_per_second": 52.1516951228585, |
| "overall": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_q8_remaining": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_remaining_gaps_v2.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 6, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 1.0, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.142813170037698, |
| "examples_per_second": 42.01293198950906, |
| "overall": { |
| "precision": 0.75, |
| "recall": 0.6, |
| "f1": 0.6666666666666665, |
| "tp": 3, |
| "fp": 1, |
| "fn": 2 |
| }, |
| "by_label": { |
| "PASSPORT_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 1, |
| "fn": 2 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "candidate_q8_multilingual": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/multilingual_ppsn_v1_all.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 168, |
| "batch_size": 16, |
| "ppsn_decoder": "word_aligned", |
| "elapsed_seconds": 0.8803652579663321, |
| "examples_per_second": 190.8298839371338, |
| "label_filter": "PPSN", |
| "overall": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 84, |
| "fp": 12, |
| "fn": 0 |
| }, |
| "by_domain": { |
| "citizen_chat": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "gov_data": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "hse_medical": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| } |
| }, |
| "by_language": { |
| "de": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "en": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "es": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "fr": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ga": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "it": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ja": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "nl": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "pl": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "pt": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ro": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ru": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "uk": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "zh": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| } |
| }, |
| "onnx_file": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8/model_quantized.onnx" |
| }, |
| "q8_recipe_compare": [ |
| { |
| "variant": "current_q8", |
| "core_f1": 0.9615384615384615, |
| "core_eps": 119.06137431000792, |
| "edge_f1": 0.9500000000000001, |
| "edge_eps": 69.96690039628702, |
| "numeric_f1": 0.8387096774193549, |
| "numeric_eps": 28.129100872864512, |
| "gap_f1": 0.8333333333333333, |
| "gap_eps": 41.070526382683134, |
| "multilingual_f1": 0.7887323943661972, |
| "multilingual_eps": 86.94930904250197, |
| "remaining_iou1_f1": 0.6, |
| "remaining_iou1_eps": 117.17196092182134 |
| }, |
| { |
| "variant": "q8_mg", |
| "core_f1": 0.9615384615384615, |
| "core_eps": 62.54552479459656, |
| "edge_f1": 0.9500000000000001, |
| "edge_eps": 100.26464304046029, |
| "numeric_f1": 0.8387096774193549, |
| "numeric_eps": 47.47899731078072, |
| "gap_f1": 0.8333333333333333, |
| "gap_eps": 71.39182685847751, |
| "multilingual_f1": 0.7368421052631579, |
| "multilingual_eps": 162.2228381957003, |
| "remaining_iou1_f1": 0.6, |
| "remaining_iou1_eps": 66.84971455324192 |
| }, |
| { |
| "variant": "q8_mg_reduce", |
| "core_f1": 0.9554140127388535, |
| "core_eps": 111.07958753347033, |
| "edge_f1": 0.9500000000000001, |
| "edge_eps": 50.58773730012114, |
| "numeric_f1": 0.7878787878787878, |
| "numeric_eps": 42.08249226466411, |
| "gap_f1": 0.7692307692307692, |
| "gap_eps": 112.74792550347136, |
| "multilingual_f1": 0.7336244541484717, |
| "multilingual_eps": 105.25341259054518, |
| "remaining_iou1_f1": 0.4000000000000001, |
| "remaining_iou1_eps": 54.19501381319339 |
| } |
| ] |
| } |
|
|
