[
{
"model_name": "ChatGPT-4o (OR)",
"model_id": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:39:35.876180",
"shots": 2,
"config": {
"model_id": "or_chatgpt_4o_latest",
"display_name": "ChatGPT-4o (OR)",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-07",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T19:52:29.249238",
"file_results": [
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:28.536350",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T19:52:29.249238",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the identical transformation from \"howling over\" to \"howling, over,\" inserting a comma after \"howling.\" This captures the core intended change with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"yonder to-day\" and the same correction \"yonder, to-day\", inserting a comma after \"yonder\". The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir",
"corrected_text": "sir?",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:32.240177",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:10:13.294512",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"howling over\" to \"howling, over\", inserting a comma after \"howling\". The core punctuation correction is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2. Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), inserting a comma after \"yonder\". This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir",
"corrected_text": "sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:32.097546",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:06:29.831306",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the core intended change\u2014adding a comma after \"howling\". There are no discrepancies in the essential edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both identify the same original text (\"yonder to-day\") and apply the same correction by inserting a comma after \"yonder\" to produce \"yonder, to-day.\" The core change is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir",
"corrected_text": "sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:32.632963",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T19:53:29.350066",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 27,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial word in the phrase \"why did he dine\" \u2192 \"Why did he dine\" (i.e., change \"why\" to \"Why\"). The prediction captures this core change directly by specifying \"why\" \u2192 \"Why\". Although it includes less surrounding context than the ground truth, it accurately reflects the essential capitalization correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones\u201d",
"corrected_text": "stones?\u201d",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:28.762796",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:11:02.947351",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the first word of the phrase (\"why did he dine\" \u2192 \"Why did he dine\"). The prediction captures the same core change by capitalizing \"why\" \u2192 \"Why\". Although it includes less surrounding context, it correctly represents the essential intended edit (capitalizing the initial word)."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones\u201d",
"corrected_text": "stones?\u201d",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:33.185073",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:07:28.755108",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "why did",
"corrected_text": "Why did",
"observed_line_number": 27,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial \"why\" in the phrase \"why did he dine\" \u2192 \"Why did he dine.\" The prediction applies the same core capitalization change to the beginning of the phrase (\"why did\" \u2192 \"Why did\"). Although it includes less surrounding context (omits \"he dine\"), it still captures the essential intended change: lowercase \"why\" to capital \"Why\" at the start of the sentence/phrase."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones\u201d",
"corrected_text": "stones?\u201d",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:35.300032",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T19:54:33.955569",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 35,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase \"if he\" to become \"If he.\" The prediction capitalizes \"if\" to \"If.\" Although the prediction includes less surrounding context (it omits \"he\"), it captures the same core change: changing the lowercase \"if\" to uppercase \"If.\" This matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth replaces the word \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The predicted edit captures the core intended change by replacing \"said\" \u2192 \"asked\". Although it omits the surrounding context (\"Clennam\"), it still represents the essential correction identified in the ground truth."
}
],
"file_id": "005"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:52.561262",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:12:01.481859",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 35,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is capitalizing the initial word in the phrase \"if he\" \u2192 \"If he\". The prediction captures the core intended change by capitalizing \"if\" \u2192 \"If\". Although it includes less surrounding context (omits \"he\"), it still represents the same essential correction: changing the lowercase initial \"if\" to capitalized \"If\"."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction are a \"replacement\" edit, so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The predicted edit captures the core intended change by replacing \"said\" with \"asked\". Although it omits the surrounding context word \"Clennam\", the essential correction (said \u2192 asked) is correctly identified, which satisfies the content criterion."
}
],
"file_id": "005"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:58.310624",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:08:18.212404",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 33,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase \"if he\" \u2192 \"If he\". The prediction captures the same essential change by capitalizing \"if\" \u2192 \"If\". Although the prediction includes less surrounding context (it omits \"he\"), it correctly represents the core intended correction: changing the lowercase \"i\" in \"if\" to uppercase."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the predicted edit are of type \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the word \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the core intended change by replacing \"said\" with \"asked\". Although it omits the surrounding context (\"Clennam\"), that additional context is not required as long as the essential substitution is correctly identified. Therefore, the text content matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:57.695692",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T19:55:25.093056",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 10,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original_text (\"about and\") and the same corrected_text (\"about, and\"), capturing the core change of inserting a comma after \"about\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a comma after \"said\" (\"said\" \u2192 \"said,\"). This captures the essential correction exactly. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 31,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core correction: inserting a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The essential punctuation change is identical. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:59.992485",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:12:46.895163",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 10,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"about and\") and the same corrected text (\"about, and\"), capturing the identical core change of inserting a comma after \"about.\" Line numbers are to be ignored, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after \"said\" in the phrase \"said when\" \u2192 \"said, when\". The prediction captures the same core punctuation change by editing \"said\" \u2192 \"said,\". Although it omits the following word \"when\" from the span, it still reflects the essential intended correction: adding a comma immediately after \"said\"."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same corrected text (\"Clennam, in\"), capturing the identical core change: inserting a comma after \"Clennam\". Line numbers are ignored as instructed, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:59.606593",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:09:20.072201",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 10,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change \"about and\" to \"about, and\" by inserting a comma after \"about\". The core punctuation correction is identical. Line number differences are to be ignored, so the prediction captures the same intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"said\" in the phrase \"said when\" \u2192 \"said, when\". The predicted edit \"said\" \u2192 \"said,\" captures the same core punctuation insertion (a comma immediately after \"said\"). Although the prediction includes less surrounding context (it omits \"when\"), it still represents the essential intended change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 28,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The original and corrected text strings match the ground truth\u2019s intention exactly. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:37:59.823095",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T19:56:16.618231",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the edit as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates that the phrase should end with a period (\"Sun and Shadow\" \u2192 \"Sun and Shadow.\"). The prediction also results in the same corrected form ending with a period (\"Sun and Shadow,\" \u2192 \"Sun and Shadow.\"). Although the predicted original_text includes a comma that is not present in the ground truth original_text, the core intended correction\u2014ensuring the title ends with a period\u2014is accurately captured."
}
],
"file_id": "007"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:06.260232",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:13:32.142566",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:05.352935",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:10:04.794978",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to make the title end with a period (\"Sun and Shadow\" \u2192 \"Sun and Shadow.\"). The prediction also results in the same corrected text ending with a period (\"Sun and Shadow,\" \u2192 \"Sun and Shadow.\"). Although the predicted original text includes a comma that is not present in the ground truth original, the core intended correction\u2014ensuring the phrase ends with a period\u2014is correctly captured."
}
],
"file_id": "007"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:23.414022",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T19:57:58.836034",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone.",
"corrected_text": "Saone. The",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Saone it",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "man, slowly",
"corrected_text": "man, slowly",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:40.036022",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:14:26.737061",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "stream",
"corrected_text": "stream,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Saone it",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "soddenwith",
"corrected_text": "soddened with",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:41.430566",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:11:01.530582",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "soddenwith",
"corrected_text": "soddened with",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are labeled as a \"replacement,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth\u2019s core intended change is to replace \"sodden\" with \"soddened\" (i.e., add \"-ed\"). The prediction performs that same core change within a slightly larger span: it changes \"soddenwith\" to \"soddened with.\" Although the prediction also introduces a space before \"with\" and includes \"with\" in the replacement span, it still captures the essential correction from \"sodden\" \u2192 \"soddened,\" which is the ground truth\u2019s intention. Additional surrounding context is allowed, so this is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone.",
"corrected_text": "Saone. The",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "trees",
"corrected_text": "-trees",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Saone it",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:26.865102",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:15:23.114331",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "said Clennam,",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Oh! I have very much to ask of you.",
"corrected_text": "That's well! I hoped so; I am not disappointed.",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hoped so",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:26.649510",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:12:02.754305",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "thanked him",
"corrected_text": "thanked Mr. Clennam",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam",
"corrected_text": "said Clennam,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hoped so",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:26.262375",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:16:10.278372",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 10,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction are labeled as \"punctuation,\" so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" in the phrase \"hers and\" \u2192 \"hers, and\". The prediction also inserts a comma after \"hers\" (\"hers\" \u2192 \"hers,\"). Although it does not include the following word \"and,\" it captures the core intended punctuation change (adding the comma after \"hers\"). Therefore, the text content matches the ground truth's intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful",
"corrected_text": "beautiful,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that papa had abandoned",
"corrected_text": "That papa had abandoned",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "that",
"corrected_text": "That",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:38.232182",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:12:59.253934",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 10,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after the word \"hers\" in the phrase \"hers and\" (i.e., \"hers and\" \u2192 \"hers, and\"). The predicted edit \"hers\" \u2192 \"hers,\" captures the same core punctuation insertion (adding a comma immediately after \"hers\"). Although the prediction omits the following word \"and\" from the original/corrected snippets, the essential intended correction\u2014comma insertion after \"hers\"\u2014is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"moment\" (\"moment with\" \u2192 \"moment, with\"). The prediction also inserts a comma after \"moment\" (\"moment\" \u2192 \"moment,\"). Although the prediction uses a shorter text span (it omits the following word \"with\"), it captures the essential intended edit: adding a comma immediately after \"moment.\" \u0644\u0630\u0644\u0643 the core change is correctly represented."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "beautiful",
"corrected_text": "beautiful,",
"observed_line_number": 23,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are labeled as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"beautiful\" in the phrase \"beautiful that\" (i.e., \"beautiful that\" \u2192 \"beautiful, that\"). The predicted edit captures the same core punctuation change by converting \"beautiful\" \u2192 \"beautiful,\". Although it omits the following word \"that\", it still correctly represents the essential intended correction: adding a comma immediately after \"beautiful\"."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that papa had abandoned",
"corrected_text": "That papa had abandoned",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "that",
"corrected_text": "That",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly",
"corrected_text": "directly,",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:41.584870",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-08T05:35:28.497145",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 10,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1. Edit Type Accuracy: Both the ground truth and predicted edits are of type \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change inserts a comma after \"hers\" in the phrase \"hers and\" \u2192 \"hers, and\". The predicted edit \"hers\" \u2192 \"hers,\" captures the same core punctuation insertion (adding a comma immediately after \"hers\"). Although it does not include the following word \"and\" in its span, it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"moment\" in the phrase \"moment with\" \u2192 \"moment, with.\" The prediction captures the same core punctuation change by editing \"moment\" \u2192 \"moment,\" (it simply omits the following word \"with\" as context). Since the essential intended change\u2014adding a comma immediately after \"moment\"\u2014is correctly represented, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "beautiful",
"corrected_text": "beautiful,",
"observed_line_number": 23,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction are labeled as \"punctuation\", so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after the word \"beautiful\" (\"beautiful that\" \u2192 \"beautiful, that\"). The predicted edit (\"beautiful\" \u2192 \"beautiful,\") captures the same core punctuation insertion at the same location, just with less surrounding context. This aligns with the ground truth's intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that papa had abandoned",
"corrected_text": "That papa had abandoned",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "that",
"corrected_text": "That",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:39.346560",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:16:54.208008",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "whatever",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "within-doors",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:55.334908",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:13:53.490568",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "whatever",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "within-doors",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:58.438322",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:17:46.240199",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both indicate the same core change: \"city\" is corrected to \"City\". There is no mismatch in the original or corrected text, and the essential capitalization change is captured precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:39:35.876180",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-08T05:37:28.019018",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change the same word from \"city\" to \"City\". This captures the identical core correction (capitalizing the first letter) with no discrepancy in original or corrected text.\n\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction adds a comma after \"existence\" within a slightly longer context (\"existence at\" \u2192 \"existence, at\"). This captures the same essential punctuation insertion at the same target word, merely including extra surrounding text. Therefore, the core intended edit is correctly represented."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 13,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change, inserting a comma immediately after \"motionless\" in the phrase \"motionless when\" (\"motionless when\" \u2192 \"motionless, when\"). The additional context word \"when\" does not alter the essential edit. Therefore, the prediction captures the intended punctuation correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 14,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma immediately after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change\u2014adding a comma after \"view\"\u2014while including extra surrounding context (\"view by\" \u2192 \"view, by\"). This preserves the essential intended correction.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:55.866758",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:18:36.035799",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "snorts",
"corrected_text": "snorts.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:39:02.521511",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:15:36.604608",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "snorts",
"corrected_text": ".",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:39:03.100398",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-08T05:38:19.301501",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "snorts",
"corrected_text": "snorts.",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:38:57.760531",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T22:19:20.710847",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me, not",
"corrected_text": "me; not",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:39:05.732908",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-07T23:16:37.161218",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me, not",
"corrected_text": "me; not",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_chatgpt_4o_latest",
"date": "2026-02-22T15:39:04.014580",
"annotator": "ChatGPT-4o (OR)",
"annotation_date": "2025-04-08T05:39:09.525327",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "me, not",
"corrected_text": "me, ^not",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bed room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2024-05-13",
"release_source": "https://openai.com/index/gpt-4o-and-more-tools-to-chatgpt-free/",
"release_notes": "Rolling alias; using the initial public GPT-4o launch date for ChatGPT."
},
{
"model_name": "Claude Fable 5",
"model_id": "claude-fable-5",
"date": "2026-07-23T15:16:05.088580",
"shots": 2,
"config": {
"model_id": "claude-fable-5",
"display_name": "Claude Fable 5",
"shots": 2,
"temperature": 0.0,
"reasoning_effort": "high",
"date": "2026-07-23",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:13:07.890300",
"file_results": [
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:18.626108",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:13:07.890300",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both apply the same core change\u2014adding a period to the end of \"Fellow Travellers\" to make it \"Fellow Travellers.\" The original and corrected texts match the ground truth intention exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the core intended change\u2014adding a comma after \"howling\". No discrepancies are present.\n\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same correction (\"yonder, to-day\"), inserting a comma after \"yonder\". The core punctuation change is captured identically.\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction specify the same change from \"Sir\" to \"sir,\" capturing the identical core correction (lowercasing the word). Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The predicted change matches the ground truth precisely, inserting a comma after \"pockets\" to change \"pockets and\" to \"pockets, and.\" This captures the core intended correction with identical original and corrected text."
}
],
"file_id": "003"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:13.165209",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:24:55.465063",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have the type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: adding a period at the end of \"Fellow Travellers\" to make it \"Fellow Travellers.\". The original and corrected texts match the ground truth\u2019s intended punctuation change with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same correction (inserting a comma: \"yonder, to-day\"). The core change\u2014adding a comma after \"yonder\"\u2014is captured identically."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"Sir\" to \"sir\", capturing the identical core change (lowercasing the word). No mismatch in original or corrected text content."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:13.713085",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:24:50.088150",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The core intended punctuation change is captured exactly with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core punctuation insertion (adding a comma after \"howling\"). Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both apply the same core change: inserting a comma after \"yonder\" in the phrase \"yonder to-day\" to make it \"yonder, to-day\". The original and corrected texts are identical between ground truth and prediction, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "to-day, Sir",
"corrected_text": "to-day, sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth edit changes \"Sir\" to \"sir\". The predicted edit changes \"to-day, Sir\" to \"to-day, sir,\" which includes extra surrounding context but preserves the same core correction: lowercasing \"Sir\" to \"sir.\" Therefore, the essential intended change is correctly captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:13.857922",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:12:59.509714",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both edits change \"No sir\" to \"No, sir\" by inserting a comma after \"No.\" This captures the same core punctuation correction with identical original and corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"why did he dine",
"corrected_text": "\"Why did he dine",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth changes the initial word from \"why\" to \"Why\" in the phrase \"why did he dine\". The prediction applies the same core capitalization change (\"why\" \u2192 \"Why\") while including an extra leading quotation mark as additional context in both the original and corrected text. Since the essential intended change (capitalizing the initial 'w' in 'why') is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!\"",
"corrected_text": "cold stones?\"",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the ending punctuation from an exclamation mark to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction captures the same core change (! \u2192 ?) and merely includes an additional trailing quote character in both the original and corrected strings (\"cold stones!\\\"\" \u2192 \"cold stones?\\\"\"). This added context does not alter the essential edit, so the intended correction is correctly represented."
}
],
"file_id": "004"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:15.282018",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:25:03.699117",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"No sir\" to \"No, sir\" (inserting a comma after \"No\"), which captures the exact core change intended."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\u201cwhy did he dine",
"corrected_text": "\u201cWhy did he dine",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial \"w\" in \"why did he dine\" to \"Why did he dine.\" The prediction performs the same core change (\"why\" \u2192 \"Why\") while including an opening quotation mark as additional surrounding context (\"\u201cwhy did he dine\" \u2192 \"\u201cWhy did he dine\"). The quotation mark is unchanged and does not alter the essential intended correction.\n\nTherefore, both the edit type and the core textual change match the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the cold stones!\u201d",
"corrected_text": "the cold stones?\u201d",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation mark to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction performs the same core change (\"...stones!\" \u2192 \"...stones?\") while including additional surrounding context (\"the\" before the phrase and a closing quotation mark). This extra context does not alter the essential intent of the correction, which is the !\u2192? change.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
}
],
"file_id": "004"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:21.435218",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:24:51.628705",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"No sir",
"corrected_text": "\"No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"No\" in \"No sir\" \u2192 \"No, sir.\" The prediction applies the same core punctuation change, but includes an additional leading quotation mark as extra context: \"No sir \u2192 \"No, sir. This still captures the essential intended correction (adding the comma after \"No\")."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"why did he dine",
"corrected_text": "\"Why did he dine",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"capitalization\", which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth changes the initial word from \"why\" to \"Why\". The prediction applies the same core change (\"why\" \u2192 \"Why\") while including an additional leading quotation mark in both the original and corrected strings (\"\\\"why did he dine\" \u2192 \"\\\"Why did he dine\"). This added context does not alter the essential intended edit, which is capitalizing the first word.\n\nTherefore, the prediction correctly captures the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the cold stones!\"",
"corrected_text": "the cold stones?\"",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation at the end of the phrase from an exclamation point to a question mark (\"cold stones!\" \u2192 \"cold stones?\"). The prediction applies the same core change (\"...stones!\" \u2192 \"...stones?\") while including additional surrounding context (the preceding word \"the\" and a trailing quotation mark). This added context does not alter the essential correction, and the intended punctuation change is captured correctly."
}
],
"file_id": "004"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:11.093100",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:13:02.312553",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"if he\" to \"If he\", capturing the identical core correction (capitalizing the initial 'i'). No discrepancy in original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing \"said Clennam\" with \"asked Clennam\". The original and corrected text are identical between ground truth and prediction, so the core change is correctly captured."
}
],
"file_id": "005"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:13.860802",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:24:57.178355",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "do? if he",
"corrected_text": "do? If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n2) Text Content Accuracy: The ground truth change is capitalizing \"if\" to \"If\" in the phrase \"if he\" \u2192 \"If he.\" The prediction includes extra surrounding context (\"do?\"), but it performs the same core change: \"do? if he\" \u2192 \"do? If he.\" This correctly captures the intended capitalization correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is a \"replacement\", and the predicted edit type is also \"replacement\", so the type matches exactly.\n\nText Content Accuracy: Both edits replace the exact same original text (\"said Clennam\") with the exact same corrected text (\"asked Clennam\"). The core intended change is identical in the prediction and the ground truth.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
}
],
"file_id": "005"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:18.467589",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:24:53.194478",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "do? if he",
"corrected_text": "do? If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth changes \"if he\" to \"If he\" (capitalizing the initial 'i'). The prediction changes \"do? if he\" to \"do? If he,\" which includes additional surrounding context but preserves the same core correction\u2014capitalizing \"if\" to \"If\". Therefore, it captures the essential intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n\nText Content Accuracy: Both specify the same original phrase (\"said Clennam\") and the same corrected phrase (\"asked Clennam\"). The essential change from \"said\" to \"asked\" is captured identically, with no discrepancies.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
}
],
"file_id": "005"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:54.878572",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:12:59.694252",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "broad high road paved",
"corrected_text": "broad high-road paved",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is converting \"high road\" to \"high-road\" (inserting a hyphen between \"high\" and \"road\"). The predicted edit applies the same core change within a longer phrase: \"broad high road paved\" \u2192 \"broad high-road paved.\" Including extra surrounding context is acceptable, and the essential modification (adding the hyphen) is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"materials\" so that \"materials costing\" becomes \"materials, costing.\" The core punctuation correction is identical with no deviation."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "tossed about and came",
"corrected_text": "tossed about, and came",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"about\" (\"about and\" \u2192 \"about, and\"). The predicted edit also inserts a comma after \"about\" in the phrase \"tossed about and came\" \u2192 \"tossed about, and came\". Although the prediction includes additional surrounding words, it captures the same core punctuation change intended by the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the types match exactly.\n\n2) Text Content Accuracy: Both edits change the phrase from \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly\". The original and corrected texts match the ground truth\u2019s core intended change exactly.\n\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"said\" in the phrase \"said when\" \u2192 \"said, when.\" The prediction applies the same core punctuation change (inserting a comma after \"said\") but includes additional surrounding context (\"always\" before the phrase): \"always said when\" \u2192 \"always said, when.\" This preserves the essential intended correction.\n\nTherefore, both the edit type and the core text change match the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction adds the same comma after \"said\" but includes extra surrounding context (\"and said\" \u2192 \"and said,\"). This still captures the core intended punctuation edit (inserting a comma immediately after \"said\")."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 31,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"Clennam\" in the phrase \"Clennam in\" \u2192 \"Clennam, in\". The prediction applies the same core punctuation change in a longer snippet: \"Clennam in amazement\" \u2192 \"Clennam, in amazement\". This includes additional surrounding context but preserves the essential intended edit (adding a comma after \"Clennam\" before \"in\").\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did not",
"corrected_text": "association, did not",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:52.620846",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:24:58.606254",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road\", capturing the identical core change (inserting a hyphen). Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change of inserting a comma after \"materials.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and came to no haven",
"corrected_text": "about, and came to no haven",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"about\" in the phrase \"about and\" (i.e., \"about and\" \u2192 \"about, and\"). The predicted edit applies the same core punctuation change\u2014adding a comma after \"about\"\u2014but includes additional surrounding context (\"came to no haven\"). Since the essential edit (inserting the comma between \"about\" and \"and\") is correctly captured, the text content matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly.\" This captures the core intended correction with identical original and corrected text."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"said\" and \"when\" (\"said when\" \u2192 \"said, when\"). The predicted edit performs the same core punctuation change in the same location, but includes extra surrounding context (\"always said when\" \u2192 \"always said, when\"). This still captures the essential intended correction (adding the comma after \"said\"), so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction adds a comma after the same target word but includes extra surrounding context (\"and said\" \u2192 \"and said,\"). This still captures the core intended punctuation change (comma insertion after \"said\")."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 32,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"Clennam\" in the phrase \"Clennam in\" \u2192 \"Clennam, in.\" The prediction applies the same core punctuation change in a longer span: \"Clennam in amazement\" \u2192 \"Clennam, in amazement.\" Including additional surrounding words is acceptable, and the essential edit (comma insertion after \"Clennam\") matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did not mend",
"corrected_text": "association, did not mend",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:10.350187",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:25:12.537238",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction change \"high road\" to \"high-road,\" capturing the same core punctuation change (inserting a hyphen). No discrepancy in the original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the identical change from \"materials costing\" to \"materials, costing\", correctly inserting a comma after \"materials\". The predicted edit captures the same core punctuation correction as the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and came",
"corrected_text": "about, and came",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"about\" (\"about and\" \u2192 \"about, and\"). The prediction also inserts a comma after \"about\" (\"about and came\" \u2192 \"about, and came\"). Although the prediction includes extra surrounding context (the additional word \"came\"), it preserves the same core punctuation change at the same location, matching the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association, did not",
"corrected_text": "association did not",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change removes the comma after \u201cassociation\u201d (\"association, did\" \u2192 \"association did\"). The prediction applies the same core punctuation change\u2014removing the comma after \u201cassociation\u201d\u2014but includes extra surrounding context (\u201cdid not\u201d). Since the essential correction (comma deletion) is preserved and additional context is allowed, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"earnestly telling\" to \"earnestly, telling\", inserting a comma after \"earnestly\". The core punctuation correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\n\n2) Text Content Accuracy: The ground truth change inserts a comma between \"said\" and \"when\" (\"said when\" \u2192 \"said, when\"). The prediction performs the same core punctuation change, adding extra surrounding context (\"always said when\" \u2192 \"always said, when\"). The essential edit\u2014comma insertion after \"said\"\u2014is correctly captured."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The predicted edit adds a comma after \"said\" as well, but includes extra preceding context (\"and said\" \u2192 \"and said,\"). This preserves the core intended correction\u2014adding the comma after \"said\"\u2014and the additional context is acceptable under the evaluation criteria."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The prediction likewise inserts a comma after \"Clennam\" while including additional surrounding context (\"Clennam in amazement\" \u2192 \"Clennam, in amazement\"). This captures the same core punctuation correction intended by the ground truth."
}
],
"file_id": "006"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:37.315139",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:13:02.183808",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Sun and Shadow\" to make it \"Sun and Shadow.\" The prediction\u2019s extra note (\u201cPeriod here?\u201d) does not alter the edit itself and is consistent with the intended correction. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:34.820659",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:24:51.485824",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period. The predicted notes add extra context but do not alter the core correction. Therefore, the prediction captures the same essential punctuation edit as the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:43.928871",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:24:50.578900",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits use the type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. The added notes in the prediction do not alter the core correction. Therefore, the predicted edit captures the same essential change as the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:12:59.133289",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:13:24.845311",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are of type \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth corrects \"Saone\" to \"Sa\u00f4ne\" (adding the circumflex). The predicted edit corrects \"river Saone\" to \"river Sa\u00f4ne\", which includes additional surrounding context but makes the same essential change to the word \"Saone\" \u2192 \"Sa\u00f4ne\". This accurately captures the core intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth corrects \"Saone\" to \"Sa\u00f4ne\" (adding the circumflex). The prediction changes \"river Saone\" to \"river Sa\u00f4ne,\" which includes extra surrounding context but preserves the same core correction to the word \"Saone.\" Therefore, the essential intended change is accurately captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma after \"man\" in \"man slowly\" \u2192 \"man, slowly.\" The prediction performs the same core punctuation change, but with additional preceding context: \"One man slowly\" \u2192 \"One man, slowly.\" This extra context does not alter the essential edit, and the intended comma insertion is correctly captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"Chalons\" in the phrase \"Chalons was\" to make \"Chalons, was\". The core punctuation change is identical. Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:00.489443",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:25:23.835379",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"Let Loose\" to \"Let Loose.\" (adding a period). The core punctuation correction is identical in original and corrected text. Therefore, the prediction correctly captures the ground truth intent."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"dull autumn\" to \"dull, autumn\", inserting a comma after \"dull\". The core change is identical, with no discrepancies in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"man\" in the phrase \"man slowly\" \u2192 \"man, slowly.\" The prediction applies the same core punctuation change (adding a comma after \"man\") but includes extra surrounding context (\"One\" before it): \"One man slowly\" \u2192 \"One man, slowly.\" This captures the essential intended edit accurately."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly. Text Content Accuracy: Both edits change \"Chalons was\" to \"Chalons, was,\" inserting a comma after \"Chalons.\" The core change and surrounding text match precisely, so the prediction captures the intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees against",
"corrected_text": "poplar-trees, against",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with wet",
"corrected_text": "soddened with wet",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:17.066368",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:24:57.201789",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The core intended punctuation change is captured exactly with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "late, dull autumn",
"corrected_text": "late, dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"dull\" and \"autumn\" (\"dull autumn\" \u2192 \"dull, autumn\"). The predicted edit, while including additional surrounding context (\"late, dull autumn\" \u2192 \"late, dull, autumn\"), performs the same essential change: it adds a comma after \"dull\" before \"autumn\". The extra context does not alter the core correction, so the prediction captures the intended edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"Saone\" to \"Sa\u00f4ne\" (adding the circumflex). The prediction changes \"river Saone\" to \"river Sa\u00f4ne\", which includes extra surrounding context but preserves the same essential correction on the target word. Therefore, it captures the core intended change."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the type matches exactly. 2) Text Content Accuracy: The core intended change is adding the circumflex to \"Saone\" \u2192 \"Sa\u00f4ne\". The prediction applies the same change, but includes extra surrounding context (\"river \"). This still captures the essential correction on the target word, so the text content matches the ground truth's intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The core intended change is inserting a comma after the word \"man\" in the phrase \"man slowly\" \u2192 \"man, slowly.\" The prediction applies the same comma insertion, but includes extra surrounding context (\"One\"), i.e., \"One man slowly\" \u2192 \"One man, slowly.\" This preserves the essential edit and matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change\u2014adding a comma after \"Chalons\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees against",
"corrected_text": "poplar-trees, against",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:06.989215",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:13:14.396366",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same change, inserting a comma after \"heart\" to convert \"heart so\" into \"heart, so.\" The core intended correction is fully and exactly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did",
"corrected_text": "(as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"insertion\" and the predicted type is also \"insertion\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts the word \"indeed\" into the phrase \"as she did\" to make \"as indeed she did\". The prediction represents the same core insertion, simply including an extra preceding parenthesis as additional context: \"(as she did\" \u2192 \"(as indeed she did\". Since the essential modification (inserting \"indeed\") is correctly captured, the text content matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"length\" in the sequence \"length said\" \u2192 \"length, said.\" The prediction performs the same core change (inserting a comma after \"length\") but includes additional surrounding context (\"at\"). This added context does not alter the essential correction, and the intended punctuation insertion is accurately captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change from present to past tense. No discrepancies in the essential edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:09.185836",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:25:04.740827",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"heart\" in the phrase \"heart so\" to make \"heart, so.\" The original and corrected text match the ground truth intention with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did",
"corrected_text": "(as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"insertion\" and the predicted edit type is also \"insertion\", so the type matches exactly.\n\n2) Text Content Accuracy: The core intended change in the ground truth is inserting the word \"indeed\" into the phrase, changing \"as she did\" to \"as indeed she did\". The prediction captures the same core insertion, showing \"(as she did\" \u2192 \"(as indeed she did\". The added leading parenthesis is extra surrounding context included on both sides and does not alter the essential edit. Therefore, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The predicted edit performs the same core change, just with additional surrounding context (\"at length said\" \u2192 \"at length, said\"). The essential punctuation insertion is identical in intention and effect.\n\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change from present to past tense. No discrepancies in the essential content change."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:17.593458",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:24:59.796643",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change, inserting a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The core punctuation correction is identical. Therefore, the prediction correctly captures the ground truth intent."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did",
"corrected_text": "(as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"as she did\" to \"as indeed she did\" (inserting \"indeed\"). The prediction changes \"(as she did\" to \"(as indeed she did,\" which includes an extra opening parenthesis as surrounding context but preserves the same core insertion of \"indeed\" in the correct location. Since added context is allowed and the essential change is captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"length\" (\"length said\" \u2192 \"length, said\"). The prediction applies the same punctuation change within a slightly larger context (\"at length said\" \u2192 \"at length, said\"). This preserves the core intended correction\u2014adding a comma after \"length\"\u2014and the added surrounding word \"at\" does not alter the essential edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change from present to past tense. No discrepancies in the essential edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:28.552555",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:13:45.996590",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change\u2014adding a comma after \"hers.\" Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change\u2014adding a comma after \"moment\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"beautiful\" in the phrase \"beautiful that\" \u2192 \"beautiful, that\". The core intended punctuation correction is captured identically.\nTherefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they are to be married.\"",
"corrected_text": "\"They are to be married.\"",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"capitalization\" and the predicted type is also \"capitalization\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is specifically lowercased \"they\" \u2192 capitalized \"They\". The prediction applies the same core change but includes additional surrounding context (the full quoted sentence). The essential correction\u2014capitalizing the initial \"they\" to \"They\"\u2014is correctly captured."
}
],
"file_id": "010"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:31.936874",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:25:45.775095",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"hers and\" to \"hers, and\" by inserting a comma after \"hers\". The core punctuation change is identical, with matching original and corrected text. Therefore, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"hers\" to convert \"hers matters\" into \"hers, matters.\" The core punctuation correction is identical. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"moment with\" to \"moment, with,\" inserting a comma after \"moment.\" The core correction is identical, with no discrepancy in original or corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"beautiful\" in the phrase \"beautiful that\" \u2192 \"beautiful, that\". The core punctuation correction is identical.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\u201cthey are to be married.\u201d",
"corrected_text": "\u201cThey are to be married.\u201d",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so this matches exactly.\n\n2. Text Content Accuracy: The ground truth corrects the word \"they\" to \"They\". The prediction changes the quoted phrase from \u201cthey are to be married.\u201d to \u201cThey are to be married.\u201d, which includes the same core change (capitalizing the initial \"they\" to \"They\"). The additional surrounding context does not alter the essential correction, so the prediction captures the intended edit."
}
],
"file_id": "010"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:39.078792",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:25:41.259454",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"hers and\" to \"hers, and\", inserting a comma after \"hers\". The core change is identical, with no discrepancies in original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters little",
"corrected_text": "on hers, matters little",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The prediction also inserts a comma after \"hers\" within a larger phrase (\"on hers matters little\" \u2192 \"on hers, matters little\"). Although the prediction includes additional surrounding words, it captures the same core punctuation change intended by the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "for a moment with",
"corrected_text": "for a moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n2. Text Content Accuracy: The ground truth inserts a comma after \"moment\" (\"moment with\" \u2192 \"moment, with\"). The prediction applies the same core change, adding a comma after \"moment\" within a larger phrase (\"for a moment with\" \u2192 \"for a moment, with\"). Including extra surrounding context does not alter the essential edit, so the intention is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so beautiful that",
"corrected_text": "so beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"beautiful\" (\"beautiful that\" \u2192 \"beautiful, that\"). The prediction also inserts a comma after \"beautiful\" within a slightly larger phrase (\"so beautiful that\" \u2192 \"so beautiful, that\"). This added context does not change the core edit; the essential punctuation change is captured correctly."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\u201cthey are to be married.\u201d",
"corrected_text": "\u201cThey are to be married.\u201d",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the word \"they\" to \"They\" (capitalizing the initial 't'). The prediction applies the same capitalization change but includes surrounding context (the full quoted sentence), changing \u201cthey are to be married.\u201d to \u201cThey are to be married.\u201d This captures the same core change (they \u2192 They). Therefore, the predicted edit matches the ground truth\u2019s intention."
}
],
"file_id": "010"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:38.887935",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:13:58.848993",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\" murmured",
"corrected_text": "\"Thank you,\" murmured",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation in the phrase from \"Thank you!\" to \"Thank you,\" (exclamation mark to comma). The predicted edit performs the same core change, but includes additional surrounding context (the closing quote and the word \"murmured\"). The essential intended correction\u2014replacing ! with , after \"Thank you\"\u2014is accurately captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once were",
"corrected_text": "than once were",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"that\" to \"than\". The prediction changes \"that once were\" to \"than once were\", which includes extra surrounding context but preserves the same core substitution (\"that\" \u2192 \"than\"). Therefore, it correctly captures the intended edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in our fancy",
"corrected_text": "that, in our fancy",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:13:49.839231",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:25:46.845258",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\" murmured",
"corrected_text": "\"Thank you,\" murmured",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation in the phrase from \"Thank you!\" to \"Thank you,\" (exclamation mark to comma). The predicted edit performs the same core change\u2014replacing the exclamation mark after \"Thank you\" with a comma\u2014while including additional surrounding context (quotation marks and the following word \"murmured\"). This added context does not alter the essential edit, and the intended punctuation correction is captured correctly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change: insertion of a comma after \"shore\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once were",
"corrected_text": "than once were",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches.\n2) Text Content Accuracy: The ground truth replaces \"that\" with \"than\". The prediction replaces the larger span \"that once were\" with \"than once were\", which preserves the same core change (\"that\" \u2192 \"than\") while including extra surrounding context. This correctly captures the intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:16:05.088580",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:25:42.225481",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\" murmured",
"corrected_text": "\"Thank you,\" murmured",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation in \"Thank you!\" to \"Thank you,\" (exclamation mark to comma). The predicted edit performs the same core punctuation change within a larger quoted context (\"Thank you!\" murmured \u2192 \"Thank you,\" murmured). Including extra surrounding words is acceptable, and the essential correction is identical.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change, inserting a comma to change \"shore and\" to \"shore, and\". The original and corrected texts are identical between ground truth and prediction, so the intended edit is fully captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "greater things that once were",
"corrected_text": "greater things than once were",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth core change is replacing the word \"that\" with \"than\". The predicted edit replaces the longer span \"greater things that once were\" with \"greater things than once were,\" which includes the same essential word substitution (that \u2192 than) with additional surrounding context. This correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and that in",
"corrected_text": "and that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "our fancy whatever",
"corrected_text": "our fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:14:11.223780",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:13:49.709258",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Chapter XXIX\" to \"Chapter XXIX.\" by adding a period at the end. The core intended punctuation change is captured identically.\nTherefore, the predicted edit correctly matches the ground truth in both type and content."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the city",
"corrected_text": "the City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is capitalizing \"city\" to \"City.\" The prediction applies the same capitalization change to the word \"city\" but includes additional surrounding context (\"the\"). Since the essential modification\u2014\"city\" \u2192 \"City\"\u2014is correctly captured, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of the same",
"corrected_text": "return to the same",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth core change is replacing the word \"of\" with \"to\". The predicted edit replaces the phrase \"return of the same\" with \"return to the same\", which includes additional surrounding context but preserves the same essential substitution (\"of\" \u2192 \"to\"). Therefore, the prediction captures the ground truth\u2019s intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"them\" from a comma to a semicolon (\"them,\" \u2192 \"them;\"). The predicted edit performs the same core change, just with extra surrounding context (\"familiar with them,\" \u2192 \"familiar with them;\"). This captures the essential intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence at the hour",
"corrected_text": "existence, at the hour",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma immediately after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit also inserts a comma right after \"existence\" within a longer phrase (\"existence at the hour\" \u2192 \"existence, at the hour\"). This captures the same core punctuation correction, with additional surrounding context, which is allowed."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "it, to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The core change in the ground truth is replacing the comma after \"it\" with a semicolon (\"it,\" \u2192 \"it;\"). The prediction applies the same punctuation change in the same location, merely including extra surrounding context (\"to suppose\"). This still captures the essential intended correction."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change, inserting a comma immediately after \"motionless\" within a larger context (\"motionless when\" \u2192 \"motionless, when\"). This preserves the essential intended punctuation correction.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "stand-still, to be",
"corrected_text": "stand-still; to be",
"observed_line_number": 18,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing the comma after \"stand-still\" with a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The prediction performs the same core punctuation change, just with additional surrounding context (\"stand-still, to be\" \u2192 \"stand-still; to be\"). The essential intended edit\u2014comma to semicolon after \"stand-still\"\u2014is correctly captured."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 18,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma after \"view\" as well, using additional surrounding context (\"view by\" \u2192 \"view, by\"). This captures the same core punctuation insertion at the same location, so the text content matches the ground truth intention."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence, is the infirmity",
"corrected_text": "existence; is the infirmity",
"observed_line_number": 20,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing the comma after \"existence\" with a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction makes the same core punctuation substitution, merely including additional surrounding context (\"existence, is the infirmity\" \u2192 \"existence; is the infirmity\"). This correctly captures the essential intended edit."
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch Goes on Dreaming",
"corrected_text": "Mrs. Flintwinch Goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:14:14.731850",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:26:35.271389",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Chapter XXIX\" to \"Chapter XXIX.\" by adding a period at the end. The core intended punctuation change is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the city",
"corrected_text": "the City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n2) Text Content Accuracy: The ground truth change is to capitalize \"city\" \u2192 \"City.\" The prediction applies the same core capitalization change within a slightly larger context (\"the city\" \u2192 \"the City\"). This preserves the essential intended correction (capitalizing the word \"city\") and is therefore an accurate capture of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of",
"corrected_text": "return to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces \"of\" with \"to\". The predicted edit replaces \"return of\" with \"return to\", which contains the same essential change (\"of\" \u2192 \"to\") with additional surrounding context. This correctly captures the core intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth changes the punctuation in \"them,\" to \"them;\" (comma \u2192 semicolon). The prediction applies the same core change, but includes extra surrounding context (\"familiar with\"). Since the essential punctuation change at \"them,\" \u2192 \"them;\" is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence at",
"corrected_text": "busy existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction adds a comma after \"existence\" within a longer span (\"busy existence at\" \u2192 \"busy existence, at\"). This captures the same core punctuation insertion at the correct word, merely with additional surrounding context. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it, to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth changes \"it,\" to \"it;\" (comma to semicolon after \"it\"). The prediction changes \"it, to suppose\" to \"it; to suppose,\" which includes additional surrounding context but performs the same core punctuation change at the same location (comma \u2192 semicolon after \"it\"). This correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction adds a comma in the same place, after \"motionless\", while including additional surrounding context (\"motionless when\" \u2192 \"motionless, when\"). This captures the core intended punctuation change accurately."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "a stand-still, to",
"corrected_text": "a stand-still; to",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The predicted edit makes the same core change, simply including extra surrounding context (\"a stand-still, to\" \u2192 \"a stand-still; to\"). The essential punctuation substitution (comma \u2192 semicolon immediately after \"stand-still\") is correctly captured."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change in a larger context (\"our view by\" \u2192 \"our view, by\"), which still places a comma immediately after \"view\". The extra surrounding words do not alter the essential edit.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "contracted existence, is",
"corrected_text": "contracted existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes a comma to a semicolon in \"existence,\" \u2192 \"existence;\". The prediction changes the same punctuation mark at the same word boundary, but includes extra surrounding context: \"contracted existence, is\" \u2192 \"contracted existence; is\". This captures the same core correction (comma after \"existence\" replaced with a semicolon)."
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch Goes on Dreaming",
"corrected_text": "Mrs. Flintwinch Goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:14:15.252672",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:25:43.270787",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\". The original_text and corrected_text are identical between ground truth and prediction, so the intended punctuation correction is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the city",
"corrected_text": "the City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n2) Text Content Accuracy: The ground truth change is capitalizing \"city\" to \"City.\" The prediction applies the same capitalization change to the word \"city\" within a larger phrase (\"the city\" \u2192 \"the City\"). This includes extra context but preserves the core intended change (capitalizing the same target word). Therefore, it correctly captures the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of",
"corrected_text": "return to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"of\" \u2192 \"to\". The predicted edit changes \"return of\" \u2192 \"return to\", which includes additional surrounding context but preserves the same core substitution (replacing \"of\" with \"to\"). Therefore, it captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation immediately following \"them\" from a comma to a semicolon (\"them,\" \u2192 \"them;\"). The prediction makes the same core change, but includes additional surrounding context (\"familiar with them,\" \u2192 \"familiar with them;\"). This still captures the essential intended correction (comma to semicolon after \"them\").\n\nTherefore, the predicted edit correctly reflects the ground truth edit\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 17,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction shows a comma inserted after \"existence\" in the larger phrase (\"existence at\" \u2192 \"existence, at\"). This captures the same core punctuation change, with additional surrounding context, which is allowed."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "it, to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 18,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is specifically replacing the comma after \"it\" (\"it,\") with a semicolon (\"it;\"). The predicted edit expands the span to include following context (\"it, to suppose\" \u2192 \"it; to suppose\"), but it makes the same essential punctuation change at the same location (comma after \"it\" replaced by semicolon). Extra surrounding words do not alter the core correction.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 18,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change within a larger span (\"motionless when\" \u2192 \"motionless, when\"), which correctly captures the essential punctuation insertion after \"motionless\". The extra context word \"when\" does not alter the core intended edit."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "stand-still, to be",
"corrected_text": "stand-still; to be",
"observed_line_number": 19,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation immediately following \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The predicted edit performs the same core change, simply including additional surrounding context (\"stand-still, to be\" \u2192 \"stand-still; to be\"). This preserves the essential intended correction.\n\nTherefore, the predicted edit correctly captures the ground truth edit\u2019s intention."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 19,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma immediately after \"view\" as well, while including extra surrounding context (\"view by\" \u2192 \"view, by\"). This captures the same core punctuation change intended in the ground truth."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "existence, is",
"corrected_text": "existence; is",
"observed_line_number": 21,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The core change in the ground truth is replacing the comma after \"existence\" with a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction applies the same punctuation change, merely including additional surrounding context (\"existence, is\" \u2192 \"existence; is\"). This preserves and correctly captures the essential intended correction."
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:14:14.194621",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:15:08.992569",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid it",
"corrected_text": "now, I have paid it",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The predicted edit applies the same comma insertion within a longer excerpt (\"now I have paid it\" \u2192 \"now, I have paid it\"). This includes additional surrounding context but captures the same core change\u2014adding a comma after \"now\"\u2014so the textual intention matches."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"authorize\") and the same corrected text (\"authorise\"). This captures the core intended change (American to British spelling) with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that-boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Your",
"corrected_text": "Your",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to break",
"corrected_text": "now, to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty;",
"corrected_text": "at twenty,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "him, than",
"corrected_text": "him than",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:14:12.048487",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:26:40.498231",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid it",
"corrected_text": "now, I have paid it",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The prediction likewise inserts a comma after \"now\" within a longer phrase (\"now I have paid it\" \u2192 \"now, I have paid it\"). This preserves the core intended punctuation change, only adding surrounding context, which is allowed.\n\nTherefore, the predicted edit correctly captures the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify replacing \"authorize\" with \"authorise\". This captures the same core spelling change intended by the ground truth with no discrepancies. Therefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam;",
"corrected_text": "said Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that-boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty,",
"corrected_text": "at twenty;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Your",
"corrected_text": "Your",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to break",
"corrected_text": "now, to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "out of him, than",
"corrected_text": "out of him than",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:14:12.567450",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:27:29.977656",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid it",
"corrected_text": "now, I have paid it",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The prediction also inserts a comma after \"now\" (\"now I have paid it\" \u2192 \"now, I have paid it\"). Although the prediction includes additional surrounding words, it captures the same core punctuation change and matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both replace the original text \"authorize\" with the corrected text \"authorise\". The core intended spelling change is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam; \"not that",
"corrected_text": "said Clennam, \"not that",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a little",
"corrected_text": "a little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty, sticking",
"corrected_text": "at twenty; sticking",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "bubbling out of him",
"corrected_text": "bubbling out of him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to break",
"corrected_text": "now, to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "out of him than",
"corrected_text": "out of him, than",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:14:03.645441",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T11:14:05.235234",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"For Mr. Pancks\" to \"For, Mr. Pancks\" by inserting a comma after \"For\". The core punctuation correction is identical. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". The core change is identical, with matching original and corrected text. Therefore the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me, not",
"corrected_text": "making me; not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:14:17.336212",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T12:25:52.764744",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"For Mr. Pancks\" to \"For, Mr. Pancks\" by inserting a comma after \"For\". The core punctuation correction is identical. Therefore, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"quickly that\") and the same correction (\"quickly, that\"), inserting a comma after \"quickly\". The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me, not",
"corrected_text": "making me; not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-fable-5",
"date": "2026-07-23T15:14:36.870889",
"annotator": "Claude Fable 5",
"annotation_date": "2026-07-23T13:25:46.232895",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same corrected text (\"For, Mr. Pancks\"), capturing the intended insertion of a comma after \"For\". Therefore, the core change matches the ground truth exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same change, inserting a comma after \"quickly\" (\"quickly that\" \u2192 \"quickly, that\"). The core correction is identical, with no mismatch in original or corrected text. Therefore, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me, not",
"corrected_text": "making me; not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2026-06-09",
"release_source": "https://www.anthropic.com/news/claude-fable-5-mythos-5",
"release_notes": "First model in the Claude 5 family (Mythos-class tier above Opus). Availability was suspended by a US Commerce Department order on 2026-06-12 and restored on 2026-07-01.",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Fable 5.1 (OR)",
"model_id": "or_claude_fable_5_1",
"date": "2026-09-06T04:17:14.450337",
"shots": 2,
"config": {
"model_id": "or_claude_fable_5_1",
"display_name": "Claude Fable 5.1 (OR)",
"shots": 2,
"temperature": 0.0,
"reasoning_effort": "high",
"date": "2026-09-06",
"notes": "Benchmark run with 2-shot learning at high reasoning effort"
},
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:23.140515",
"file_results": [
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:41.065702",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:23.140515",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same core punctuation correction, changing \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The essential edit is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"howling over\" to \"howling, over\" by inserting a comma after \"howling.\" The core punctuation change is identical with no discrepancy in original or corrected text.\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same correction (\"yonder, to-day\"), inserting a comma after \"yonder\". The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"Sir\" to \"sir\". The core intended change (lowercasing the initial letter) is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"pockets\" in the phrase \"pockets and\" to become \"pockets, and.\" There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
}
],
"file_id": "003"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:34.784248",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:39.377923",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same core change\u2014adding a period to \"Fellow Travellers\" to become \"Fellow Travellers.\" There are no discrepancies in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same core change: inserting a comma after \"howling\" so that \"howling over\" becomes \"howling, over\". The original_text and corrected_text match the ground truth exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same correction (inserting a comma: \"yonder, to-day\"). The core intended change\u2014adding a comma after \"yonder\"\u2014is captured identically."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"Sir\" to \"sir\". The core intended change (lowercasing the initial letter) is captured precisely with no mismatch in original or corrected text."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"pockets and\") and the same correction (\"pockets, and\"), capturing the identical core change of inserting a comma after \"pockets\". Therefore, the predicted edit matches the ground truth intention."
}
],
"file_id": "003"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:40.997287",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:13:47.355653",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. This captures the identical core punctuation correction with the same original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"howling over\" to \"howling, over\" by inserting a comma after \"howling.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"yonder to-day\" to \"yonder, to-day\" by inserting a comma after \"yonder\". The original and corrected text match the ground truth\u2019s core change exactly."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify changing \"Sir\" to \"sir\", capturing the same core capitalization correction with identical original and corrected text.\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"pockets\" (\"pockets and\" \u2192 \"pockets, and\"). The core intended punctuation correction is captured identically."
}
],
"file_id": "003"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:32.866128",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:21.171510",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text \"No sir\" and the same corrected text \"No, sir\", capturing the identical core change (inserting a comma after \"No\"). Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"why did he dine",
"corrected_text": "\"Why did he dine",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial \"w\" in \"why did he dine\" to \"Why did he dine.\" The prediction applies the same capitalization change, but includes an additional leading quotation mark in both the original and corrected strings (\"\\\"why did he dine\" \u2192 \"\\\"Why did he dine\"). This extra context does not alter the core change being captured (why \u2192 Why), so it correctly reflects the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!\"",
"corrected_text": "cold stones?\"",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation at the end of the phrase from an exclamation mark to a question mark (\"cold stones!\" \u2192 \"cold stones?\"). The predicted edit captures the same core change (\"cold stones!\\\"\" \u2192 \"cold stones?\\\"\"). The additional trailing quote mark is extra context present in both the predicted original and corrected text and does not alter the essential intended correction (! \u2192 ?). Therefore, the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "004"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:32.880755",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:43.547506",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"No sir\") and the same corrected text (\"No, sir\"), capturing the identical core change of inserting a comma after \"No.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"why did he dine",
"corrected_text": "\"Why did he dine",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial \"w\" in \"why\" to \"Why\" in the phrase \"why did he dine\". The prediction captures the same core change (\"why\" \u2192 \"Why\") and merely includes an extra leading quotation mark as additional context in both the original and corrected text. Since the essential capitalization correction is preserved, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!\"",
"corrected_text": "cold stones?\"",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation mark to a question mark (\"cold stones!\" \u2192 \"cold stones?\"). The prediction captures the same core change (! \u2192 ?) and merely includes an additional trailing quotation mark in both the original and corrected strings (\"cold stones!\\\"\" \u2192 \"cold stones?\\\"\"). This added context does not alter the essential intended correction, so the text-content requirement is satisfied."
}
],
"file_id": "004"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:36.720816",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:13:56.350345",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"No sir\" to \"No, sir\" (inserting a comma after \"No\"), which captures the core intended correction. Therefore the predicted edit matches the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"why did he dine",
"corrected_text": "\"Why did he dine",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to capitalize the initial \"why\" to \"Why\" in the phrase \"why did he dine\". The prediction makes the same core change (\"why\" \u2192 \"Why\") and merely includes an additional leading quotation mark as surrounding context in both the original and corrected text. Since the essential capitalization correction is captured correctly and extra context is allowed, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!\"",
"corrected_text": "cold stones?\"",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the terminal punctuation in \"cold stones!\" to a question mark (\"cold stones?\"). The prediction applies the same core change (! \u2192 ?) while including an additional trailing quotation mark in both original and corrected strings (\"cold stones!\\\"\" \u2192 \"cold stones?\\\"\"). This added context does not alter the essential intended edit, and the key punctuation replacement is correctly captured."
}
],
"file_id": "004"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:30.200333",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:27.162242",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"capitalization\" and the predicted type is also \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change the same phrase from \"if he\" to \"If he\", capturing the identical core capitalization correction. No mismatch in original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both change the exact same phrase from \"said Clennam\" to \"asked Clennam.\" The core intended substitution is identical with no discrepancies.\nTherefore, the predicted edit correctly captures the ground truth intention."
}
],
"file_id": "005"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:30.752161",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:45.990013",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "do? if he",
"corrected_text": "do? If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is capitalizing \"if\" to \"If\" in the phrase \"if he\" \u2192 \"If he.\" The prediction applies the same capitalization change, but includes additional surrounding context (\"do? \") in both original and corrected text: \"do? if he\" \u2192 \"do? If he.\" This still captures the core intended correction (lowercase \"if\" to uppercase \"If\")."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both identify the same original text (\"said Clennam\") and the same corrected text (\"asked Clennam\"), capturing the identical core change with no discrepancies. Therefore, the predicted edit matches the ground truth intention."
}
],
"file_id": "005"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:35.088856",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:13:49.270925",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "do? if he",
"corrected_text": "do? If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is to capitalize the initial \"if\" in the phrase \"if he\" \u2192 \"If he.\" The prediction applies the same capitalization change, but includes additional surrounding context (\"do? \"). Since the core change (lowercase \"if\" to capitalized \"If\") is correctly captured, the text content matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly. Text Content Accuracy: Both replace the exact same original text \"said Clennam\" with the corrected text \"asked Clennam\". The core change is identical and fully captures the ground truth intention."
}
],
"file_id": "005"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:44.938107",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:23.680999",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify changing \"high road\" to \"high-road\" (inserting a hyphen between the words). The original and corrected text match the ground truth core change exactly. Therefore, the predicted edit correctly captures the intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"materials costing\" to \"materials, costing\", correctly capturing the intended insertion of a comma after \"materials\". No discrepancy in the core change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly. Text Content Accuracy: Both ground truth and prediction change \"about and\" to \"about, and\" by inserting a comma after \"about.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion, did",
"corrected_text": "tion did",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change removes the comma between \"association\" and \"did\" (\"association, did\" \u2192 \"association did\"). The prediction removes the comma in the corresponding substring (\"tion, did\" \u2192 \"tion did\"). Although the prediction provides less surrounding context (truncating \"association\" to its ending), it captures the same core intended punctuation edit: deleting the comma before \"did\". Therefore the text content matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same transformation from \"earnestly telling\" to \"earnestly, telling,\" inserting a comma after \"earnestly.\" The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"said when\" to \"said, when\", inserting a comma after \"said\". The core punctuation change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction adds a comma after the phrase ending in \"said\" (\"and said\" \u2192 \"and said,\"). This includes additional preceding context (\"and\") but preserves the same core correction: inserting a comma immediately after \"said\". Therefore, the prediction captures the ground truth\u2019s intended punctuation change."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"Clennam in\") and the same corrected text (\"Clennam, in\"), capturing the identical core change: inserting a comma after \"Clennam.\" Therefore, the predicted edit matches the ground truth intention."
}
],
"file_id": "006"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:48.074385",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:42.799973",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have the type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text \"high road\" and the same correction to \"high-road\", capturing the identical core change (adding a hyphen). Therefore, the prediction correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change of inserting a comma after \"materials\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"about and\" \u2192 \"about, and\" (inserting a comma after \"about\"). This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion, did",
"corrected_text": "tion did",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth removes the comma in \"association, did\" \u2192 \"association did\". The prediction removes the comma in \"tion, did\" \u2192 \"tion did\". Although the prediction includes less surrounding context (it truncates \"association\" to its ending \"tion\"), it captures the same core change: deleting the comma immediately before \"did\". Therefore, the essential intended punctuation correction is accurately represented."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit type matches exactly: both are labeled as \"punctuation\". Text content also matches the core intended change: both convert \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly\". No discrepancy in the essential correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"said when\") and the same corrected text (\"said, when\"), capturing the identical core punctuation insertion (adding a comma after \"said\"). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction applies the same core change\u2014adding a comma after \"said\"\u2014but includes additional preceding context (\"and said\" \u2192 \"and said,\"). This preserves the essential intended correction.\n\nTherefore, the predicted edit correctly captures the ground truth edit\u2019s intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"Clennam in\" \u001f \"Clennam, in\" (inserting a comma after \"Clennam\"). The core change is identical with no discrepancies in original or corrected text."
}
],
"file_id": "006"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:39.137086",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:13:58.035636",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: replacing \"high road\" with \"high-road\" (inserting a hyphen). The original and corrected texts are identical between ground truth and prediction, capturing the intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"materials\" in the phrase \"materials costing\", resulting in \"materials, costing\". The core punctuation correction is captured identically.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits have the type \"punctuation\", so the edit type matches exactly. Text Content Accuracy: Both edits change \"about and\" to \"about, and\" by inserting a comma after \"about\". The core intended punctuation correction is captured precisely with identical original and corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), inserting a comma after \"earnestly\". The core change is identical. Therefore the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"said when\" to \"said, when\", correctly capturing the core intended edit (inserting a comma after \"said\"). Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction adds a comma after \"said\" as well, but includes extra surrounding context (\"and said\" \u2192 \"and said,\"). This still captures the same core punctuation change at the same target word, so the text content matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit type matches exactly: both are labeled as \"punctuation.\" Text content matches the core change: both edits insert a comma after \"Clennam,\" changing \"Clennam in\" to \"Clennam, in.\" Therefore the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion, did not",
"corrected_text": "tion did not",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:54.068687",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:11.460403",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to change \"Sun and Shadow\" to \"Sun and Shadow.\" The prediction\u2019s extra note about a handwritten query does not alter the edit itself and is acceptable additional context. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:59.751662",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:32.618794",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Ground truth changes \"Sun and Shadow\" to \"Sun and Shadow.\" (adds a period). The predicted edit applies the same core change with identical original and corrected text. The additional notes about a handwritten query do not alter the edit itself. Therefore, the prediction captures the intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:15:57.959102",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:13:43.174389",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW",
"corrected_text": "SUN AND SHADOW.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:07.816768",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:22.319090",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma between \"dull\" and \"autumn\" (\"dull autumn\" \u2192 \"dull, autumn\"). The original and corrected texts align perfectly with the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction are of type \"replacement\", so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"Saone\" to \"Sa\u00f4ne\" (adding the circumflex). The predicted edit changes \"river Saone\" to \"river Sa\u00f4ne\", which includes extra surrounding context but preserves the same core correction on the target word. Therefore, the essential intended change is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: The ground truth corrects \"Saone\" to \"Sa\u00f4ne\" (adding the circumflex). The predicted edit changes \"river Saone\" to \"river Sa\u00f4ne\", which includes extra surrounding context but preserves the same essential correction on the target word. Therefore, it captures the core intended change."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma after \"man\" (\"man slowly\" \u2192 \"man, slowly\"). The prediction also inserts a comma after \"man\" while including extra preceding context (\"One man slowly\" \u2192 \"One man, slowly\"). This preserves the core intended punctuation change. Therefore the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change\u2014adding a comma after \"Chalons\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:21.452185",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:44.440554",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "late, dull autumn",
"corrected_text": "late, dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma between \"dull\" and \"autumn\" (\"dull autumn\" \u2192 \"dull, autumn\"). The prediction applies the same core change within a longer context (\"late, dull autumn\" \u2192 \"late, dull, autumn\"), which includes adding the comma after \"dull\" before \"autumn\". The additional surrounding word \"late,\" does not alter the essential intended correction. Therefore, the prediction captures the ground truth\u2019s core punctuation change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"Saone\" to \"Sa\u00f4ne\" (adding the circumflex). The prediction changes \"river Saone\" to \"river Sa\u00f4ne\", which includes extra surrounding context but preserves the same essential correction to the word \"Saone\". Therefore, the core intended change is accurately captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone it",
"corrected_text": "river Sa\u00f4ne it",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth replaces \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex). The predicted edit replaces \"river Saone it\" with \"river Sa\u00f4ne it\", which includes extra surrounding context but performs the same core change: \"Saone\" \u2192 \"Sa\u00f4ne\". Therefore, it accurately captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth change inserts a comma after \"man\" in the phrase \"man slowly\" \u2192 \"man, slowly.\" The prediction applies the same core punctuation change, inserting a comma after \"man\" in a slightly longer snippet: \"One man slowly\" \u2192 \"One man, slowly.\" This includes extra context but preserves the essential intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the core intended change: insertion of a comma after \"Chalons\". Therefore the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with wet",
"corrected_text": "soddened with wet",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:17.358309",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:13:52.098112",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: The ground truth corrects \"Saone\" to \"Sa\u00f4ne\" (adding the circumflex). The predicted edit changes \"river Saone\" to \"river Sa\u00f4ne\", which includes extra surrounding context but preserves the same essential correction on the word \"Saone\". Therefore, it accurately captures the intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"Saone\" to \"Sa\u00f4ne\" (adding the circumflex). The prediction changes \"river Saone\" to \"river Sa\u00f4ne\", which includes extra surrounding context but performs the same core correction on the target word. This accurately captures the intended change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"man\" (\"man slowly\" \u2192 \"man, slowly\"). The prediction applies the same core change within a slightly larger span (\"One man slowly\" \u2192 \"One man, slowly\"). The essential punctuation insertion is identical and correctly captures the intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change\u2014adding a comma after \"Chalons.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees against",
"corrected_text": "poplar-trees, against",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:14.214762",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:36.627385",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"heart so\" to \"heart, so\" (inserting a comma after \"heart\"). The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": 22,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same change: removing the comma after \"generous\" in \"generous, Mr.\" to produce \"generous Mr.\" The original and corrected texts match the ground truth\u2019s core intended edit with no discrepancies."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did",
"corrected_text": "(as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify an \"insertion\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts the word \"indeed\" into the phrase \"as she did\" to make \"as indeed she did\". The prediction captures the same core change (inserting \"indeed\" in the same location) while including an extra leading parenthesis as additional surrounding context in both the original_text and corrected_text. This added context does not alter the essential intended edit, so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The predicted edit does the same core punctuation insertion, just with additional surrounding context (\"at length said\" \u2192 \"at length, said\"). This preserves the essential intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change (present tense \"hope\" to past tense \"hoped\"). Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "009"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:16.015617",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:55.182810",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so much",
"corrected_text": "heart, so much",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The prediction also inserts a comma after \"heart\" (\"heart so much\" \u2192 \"heart, so much\"). Although the prediction includes additional surrounding words (\"so much\"), it captures the same core punctuation change in the correct location. Therefore, it matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": 22,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same change from \"generous, Mr.\" to \"generous Mr.\", removing the comma after \"generous\". The core intended correction is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did",
"corrected_text": "(as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches.\n\n2) Text Content Accuracy: The core intended change is inserting the word \"indeed\" into the phrase \"as she did\" to make it \"as indeed she did.\" The prediction captures exactly this insertion. The only difference is that the predicted original/corrected text includes an extra leading parenthesis \"(\" in both versions, which is permissible additional context and does not alter the essential edit.\n\nTherefore, the predicted edit correctly reflects the ground truth\u2019s intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said Clennam",
"corrected_text": "at length, said Clennam",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"length\" (\"length said\" \u2192 \"length, said\"). The prediction performs the same core punctuation change within a longer phrase (\"at length said Clennam\" \u2192 \"at length, said Clennam\"). The added surrounding context does not alter the essential intended correction. Therefore, the predicted edit captures the ground truth\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change (tense shift from hope to hoped). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "009"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:20.286982",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:13:59.879140",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so much",
"corrected_text": "heart, so much",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The predicted edit also inserts a comma after \"heart\" (\"heart so much\" \u2192 \"heart, so much\"). Although the prediction includes extra surrounding context (adding \"much\"), it preserves the same core punctuation correction at the same location. Therefore, it captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did",
"corrected_text": "(as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as an \"insertion,\" so the type matches exactly.\n\n2) Text Content Accuracy: The core intended change is inserting the word \"indeed\" into the phrase \"as she did\" to make it \"as indeed she did.\" The predicted edit captures this exact insertion. The prediction includes an additional preceding parenthesis in both the original and corrected text (\"(as ...\"), which is allowable extra context and does not alter the essential change. Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said Clennam",
"corrected_text": "at length, said Clennam",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma between \u201clength\u201d and \u201csaid\u201d (\"length said\" \u2192 \"length, said\"). The predicted edit applies the same core punctuation change, adding a comma after \u201clength,\u201d but includes additional surrounding context (\"at length said Clennam\" \u2192 \"at length, said Clennam\"). This extra context does not alter the essential intended correction, which is the comma insertion after \u201clength.\u201d"
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change from present to past tense. No mismatch or missing content is present."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:22.064436",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:27.078852",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change\u2014adding a comma after \"hers\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"moment with\" to \"moment, with\", inserting a comma after \"moment\". The core change is identical, with no discrepancies in wording or punctuation."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the identical change from \"beautiful that\" to \"beautiful, that\"\u2014inserting a comma after \"beautiful.\" The core punctuation correction is captured precisely with matching original and corrected text."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they are to be married.\"",
"corrected_text": "\"They are to be married.\"",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth changes the word \"they\" to \"They\" (capitalizing the initial 't'). The prediction applies the same capitalization change to \"they\" at the start of the quoted phrase (\"they are to be married.\" \u2192 \"They are to be married.\"). Although the prediction includes additional surrounding context (the rest of the sentence and quotation marks), it captures the same core change intended by the ground truth."
}
],
"file_id": "010"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:33.008957",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:44.287954",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "from hers and",
"corrected_text": "from hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"hers\" (\"hers and\" \u2192 \"hers, and\"). The prediction also inserts a comma after \"hers,\" but includes extra surrounding context (\"from hers and\" \u2192 \"from hers, and\"). This still captures the same core punctuation correction intended by the ground truth.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters little",
"corrected_text": "on hers, matters little",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The prediction performs the same essential change\u2014adding a comma immediately after \"hers\"\u2014but includes extra surrounding context (\"on\" before and \"matters little\" after). This added context does not alter the core punctuation correction, so the intended edit is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "for a moment with",
"corrected_text": "for a moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"moment\" (\"moment with\" \u2192 \"moment, with\"). The prediction performs the same core punctuation insertion, merely including extra surrounding context (\"for a moment with\" \u2192 \"for a moment, with\"). This captures the essential intended edit accurately."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so beautiful that",
"corrected_text": "so beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \u201cbeautiful\u201d in the phrase \u201cbeautiful that\u201d \u2192 \u201cbeautiful, that.\u201d The prediction performs the same core punctuation change (comma after \u201cbeautiful\u201d) but includes additional surrounding context (\u201cso\u201d before the phrase): \u201cso beautiful that\u201d \u2192 \u201cso beautiful, that.\u201d This preserves the essential intended correction.\n\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they are to be married.\"",
"corrected_text": "\"They are to be married.\"",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change \"they\" to \"They\" (capitalizing the initial 't'). The prediction performs the same core change, but includes additional surrounding context (the full quoted sentence), changing \"\\\"they are to be married.\\\"\" to \"\\\"They are to be married.\\\"\". This captures the essential capitalization correction identified in the ground truth.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:32.999047",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:13:54.851299",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "from hers and",
"corrected_text": "from hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" in the phrase \"hers and\" \u2192 \"hers, and\". The prediction applies the same core punctuation change, but includes extra preceding context: \"from hers and\" \u2192 \"from hers, and\". This still captures the essential intended edit (adding the comma after \"hers\").\n\nBecause both the edit type and the core textual change align with the ground truth intention, the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters",
"corrected_text": "on hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma between \"hers\" and \"matters\" (\"hers matters\" \u2192 \"hers, matters\"). The prediction performs the same core punctuation change, merely including extra surrounding context (\"on hers matters\" \u2192 \"on hers, matters\"). The essential edit\u2014adding the comma after \"hers\"\u2014is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"moment with\" to \"moment, with,\" inserting a comma after \"moment.\" The core punctuation change is identical."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text \"beautiful that\" and the same corrected text \"beautiful, that\", inserting a comma after \"beautiful\". The core change is identical and fully matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they are",
"corrected_text": "\"They are",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"capitalization,\" which exactly matches the ground truth.\n\n2) Text Content Accuracy: The ground truth change is capitalizing \"they\" to \"They.\" The prediction applies the same capitalization change within a larger span (\"\\\"they are\" \u2192 \"\\\"They are\"). Although it includes additional surrounding context (a preceding quote and the following word \"are\"), the core intended edit\u2014changing lowercase \"they\" to capitalized \"They\"\u2014is correctly captured."
}
],
"file_id": "010"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:34.371718",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:22.367843",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\" murmured",
"corrected_text": "\"Thank you,\" murmured",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"Thank you!\" to \"Thank you,\" (replacing the exclamation mark with a comma). The predicted edit makes the same core punctuation change within a larger quoted phrase (\"Thank you!\" murmured \u2192 \"Thank you,\" murmured). The added surrounding context does not alter the essential correction, and the intended punctuation change is accurately captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same correction (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "greater things that once",
"corrected_text": "greater things than once",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as a \"replacement,\" so the edit type matches.\n\n2) Text Content Accuracy: The ground truth change is specifically replacing \"that\" with \"than.\" The predicted edit replaces the phrase \"greater things that once\" with \"greater things than once,\" which includes additional surrounding context but preserves the same core correction (\"that\" \u2192 \"than\"). Therefore, it correctly captures the intended change."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:36.406789",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:51.775093",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\" murmured",
"corrected_text": "\"Thank you,\" murmured",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation in \"Thank you!\" to \"Thank you,\" (exclamation mark to comma). The predicted edit makes the same core punctuation change, just with additional surrounding context (including quotation marks and the following word \"murmured\"). This still captures the essential intended correction.\n\nTherefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"shore and\" to \"shore, and,\" inserting a comma after \"shore.\" The core intended punctuation correction is captured with identical original and corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "things that once",
"corrected_text": "things than once",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are \"replacement\" edits, so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"that\" to \"than\". The predicted edit replaces the larger span \"things that once\" with \"things than once\", which includes the same core substitution (\"that\" \u2192 \"than\") while adding surrounding context. This still captures the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:43.462276",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:14:01.488310",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that, in our fancy",
"corrected_text": "that in our fancy",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change removes the comma in \"that, in\" to make it \"that in.\" The predicted edit applies the same core change (removing the comma after \"that\") but includes extra surrounding context (\"our fancy\"). This additional context does not alter the essential correction; it still captures the intended punctuation removal.\n\nTherefore, the predicted edit correctly reflects the ground truth's intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"fancy, whatever\") and the same corrected text (\"fancy whatever\"), capturing the core intention to remove the comma. Therefore, the predicted edit fully matches the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\" murmured",
"corrected_text": "\"Thank you,\" murmured",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation in the phrase from \"Thank you!\" to \"Thank you,\" (exclamation mark to comma). The predicted edit applies the same core change, but includes additional surrounding context (the quotation marks and the following word \"murmured\"). This added context does not alter the essential correction, which is the same punctuation replacement.\n\nTherefore, the prediction correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "things that once were",
"corrected_text": "things than once were",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to replace the single word \"that\" with \"than\". The prediction replaces the phrase \"things that once were\" with \"things than once were\", which includes the same core change (\"that\" \u2192 \"than\") while adding surrounding context. Since the essential substitution is correctly captured, the text content matches the ground truth's intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:17:08.735077",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:09:27.389165",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"Chapter XXIX\" \u2192 \"Chapter XXIX.\" (adding a period). This captures the identical core change with no discrepancy.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the city preserved",
"corrected_text": "the City preserved",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n2) Text Content Accuracy: The ground truth change is to capitalize the word \"city\" to \"City.\" The prediction applies the same core change (\"city\" \u2192 \"City\") within a larger phrase (\"the city preserved\" \u2192 \"the City preserved\"). Including surrounding context is allowed, and the essential capitalization correction is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of the same",
"corrected_text": "return to the same",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\" edits, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates changing \"of\" to \"to\". The predicted edit changes the phrase \"return of the same\" to \"return to the same,\" which contains the same core substitution (\"of\" \u2192 \"to\") with additional surrounding context. This correctly captures the intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation immediately following \"them\" from a comma to a semicolon (\"them,\" \u2192 \"them;\"). The predicted edit performs the same core change, just with additional surrounding context (\"familiar with them,\" \u2192 \"familiar with them;\"). The essential punctuation substitution is correctly captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence at the hour",
"corrected_text": "busy existence, at the hour",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth adds a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit adds a comma immediately after \"existence\" within a longer phrase (\"busy existence at the hour\" \u2192 \"busy existence, at the hour\"). Although it includes additional surrounding words, it captures the same core punctuation change intended by the ground truth."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it, to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: The ground truth changes the punctuation after \"it\" from a comma to a semicolon (\"it,\" \u2192 \"it;\"). The predicted edit makes the same core change, simply including additional surrounding context (\"it, to suppose\" \u2192 \"it; to suppose\"). The essential punctuation substitution is identical and correctly captured."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction adds the same comma after \"motionless\" while including extra surrounding context (\"motionless when\" \u2192 \"motionless, when\"). This captures the essential intended punctuation change."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still, to be",
"corrected_text": "stand-still; to be",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\n2. Text Content Accuracy: The ground truth changes the punctuation after \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The prediction makes the same core change, only including extra surrounding context (\"to be\") in both original and corrected strings. The essential intended correction (comma \u2192 semicolon immediately after \"stand-still\") is correctly captured."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma after \"view\" within a longer phrase (\"our view by\" \u2192 \"our view, by\"). This captures the same core intended punctuation insertion; the extra surrounding words are acceptable context.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "contracted existence, is",
"corrected_text": "contracted existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth core change is replacing the comma after \"existence,\" with a semicolon (\"existence;\" instead of \"existence,\"). The prediction makes the same punctuation substitution in context (\"contracted existence, is\" \u2192 \"contracted existence; is\"). Although the prediction includes extra surrounding words, it captures the essential intended change: comma-to-semicolon immediately following \"existence\"."
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:17:14.450337",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:11:51.903466",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\" The original and corrected texts are identical between ground truth and prediction, capturing the intended punctuation insertion precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the city preserved",
"corrected_text": "the City preserved",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is specifically \"city\" \u2192 \"City\". The prediction applies the same capitalization change to the word \"city\" within a longer span (\"the city preserved\" \u2192 \"the City preserved\"). Including additional surrounding context is acceptable, and the core intended change (capitalizing \"city\" to \"City\") is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of the same",
"corrected_text": "return to the same",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth specifies changing \"of\" to \"to\". The predicted edit changes \"return of the same\" to \"return to the same\", which includes the same core substitution (\"of\" \u2192 \"to\") with additional surrounding context. This correctly captures the intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth changes a comma to a semicolon after the word \"them\" (\"them,\" \u2192 \"them;\"). The predicted edit applies the same core punctuation change, just with additional surrounding context (\"familiar with them,\" \u2192 \"familiar with them;\"). Since the essential modification (comma replaced by semicolon after \"them\") is correctly captured, the text content matches the ground truth's intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence at the hour",
"corrected_text": "busy existence, at the hour",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same essential punctuation change within a larger phrase (\"busy existence at the hour\" \u2192 \"busy existence, at the hour\"), which correctly captures the intended core edit: inserting a comma immediately after \"existence\". \u0905\u0924\u093f\u0930\u093f\u0915\u094d\u0924 context does not alter the correctness of the core change."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it, to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"it,\" to \"it;\" (comma to semicolon). The predicted edit changes \"it, to suppose\" to \"it; to suppose,\" which includes extra surrounding context but preserves the same core punctuation substitution at \"it,\" \u2192 \"it;\". Therefore, the essential intended edit is correctly captured."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit also adds a comma after \"motionless\" and simply includes the following word for context (\"motionless when\" \u2192 \"motionless, when\"). This captures the same core punctuation insertion at the correct location.\n\nTherefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still, to be",
"corrected_text": "stand-still; to be",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The prediction captures the same core change, simply including additional surrounding context (\"to be\"). The essential punctuation substitution is identical, so it matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction expands the context to \"our view by\" \u2192 \"our view, by,\" which still captures the same core correction: inserting a comma immediately after \"view.\" The added surrounding words do not alter the essential punctuation change.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "contracted existence, is",
"corrected_text": "contracted existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after the word \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The predicted edit applies the same core change in a larger context (\"contracted existence, is\" \u2192 \"contracted existence; is\"), which still clearly captures the intended comma-to-semicolon replacement immediately after \"existence.\" Additional surrounding words do not alter the essential correction."
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:17:14.294451",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:14:01.754492",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both specify the same change: adding a period to \"Chapter XXIX\" to become \"Chapter XXIX.\". The core punctuation change is identical in original and corrected text.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the city preserved",
"corrected_text": "the City preserved",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"capitalization,\" which matches the ground truth exactly.\n2) Text Content Accuracy: The ground truth change is capitalizing \"city\" to \"City.\" The prediction applies the same core change (\"city\" \u2192 \"City\") while including additional surrounding context (\"the\" and \"preserved\"). This still captures the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of the same",
"corrected_text": "return to the same",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing \"of\" with \"to\". The predicted edit replaces the phrase \"return of the same\" with \"return to the same\", which includes additional surrounding context but preserves the core correction (\"of\" \u2192 \"to\") in the relevant location. Therefore, the essential intended change is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation immediately after \"them\" from a comma to a semicolon (\"them,\" \u2192 \"them;\"). The prediction applies the same core change, merely including additional surrounding context (\"familiar with them,\" \u2192 \"familiar with them;\"). This preserves the essential intended correction (comma-to-semicolon after \"them\")."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at the hour",
"corrected_text": "existence, at the hour",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates inserting a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change\u2014adding a comma immediately after \"existence\"\u2014but includes additional surrounding context (\"existence at the hour\" \u2192 \"existence, at the hour\"). This preserves the essential intended correction.\n\nTherefore, the predicted edit correctly captures the ground truth edit\u2019s intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it, to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"it\" from a comma to a semicolon (\"it,\" \u2192 \"it;\"). The prediction applies the same core change, but includes additional surrounding context (\"it, to suppose\" \u2192 \"it; to suppose\"). This added context does not alter the essential edit; it still captures the comma-to-semicolon replacement at the same point. Therefore the prediction matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to add a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change, adding a comma immediately after \"motionless\" while including extra context (\"motionless when\" \u2192 \"motionless, when\"). This preserves the essential intended punctuation edit.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still, to be",
"corrected_text": "stand-still; to be",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The prediction makes the same core change at the same location, merely including extra surrounding context (\"stand-still, to be\" \u2192 \"stand-still; to be\"). Since the essential intended correction (comma \u2192 semicolon after \"stand-still\") is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change, adding a comma immediately after \"view\" within a larger context (\"view by\" \u2192 \"view, by\"). This captures the essential intended punctuation correction, merely including extra surrounding words.\n\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "contracted existence, is",
"corrected_text": "contracted existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after the word \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction applies the same core change in a longer excerpt (\"contracted existence, is\" \u2192 \"contracted existence; is\"). Although it includes additional surrounding words, the essential correction\u2014replacing the comma after \"existence\" with a semicolon\u2014is correctly captured."
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:17:02.758871",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:10:18.871455",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits change \"Clennam:\" to \"Clennam;\", replacing a colon with a semicolon. The core intended punctuation change is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly. Text Content Accuracy: Both ground truth and prediction apply the italicization to the same word, \"him\" (original_text and corrected_text are identical aside from styling intent). The core change\u2014italicizing \"him\"\u2014is fully captured. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid it",
"corrected_text": "now, I have paid it",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The predicted edit also inserts a comma after \"now\" (\"now I have paid it\" \u2192 \"now, I have paid it\"), merely including additional surrounding context. The core punctuation change is the same and correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the edit type matches exactly.\nText Content Accuracy: Both replace the same original text \"authorize\" with the same corrected text \"authorise\". The core intended change is captured with no discrepancy.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a little",
"corrected_text": "a little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty, sticking",
"corrected_text": "twenty; sticking",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "him than",
"corrected_text": "him, than",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to break",
"corrected_text": "now, to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:17:07.077688",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:12:41.031296",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid it",
"corrected_text": "now, I have paid it",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The prediction applies the same core change within a longer span of surrounding words (\"now I have paid it\" \u2192 \"now, I have paid it\"). The essential punctuation insertion is identical, and extra context is allowed. \u0628\u0646\u0627\u0628\u0631\u0627\u06cc\u0646 the prediction captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing \"authorize\" with \"authorise\". The core change (American to British spelling) is identical in the prediction and ground truth. Therefore, the predicted edit captures the intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty, sticking",
"corrected_text": "at twenty; sticking",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to break",
"corrected_text": "now, to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam",
"corrected_text": "said Clennam;",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "out of him than",
"corrected_text": "out of him, than",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:17:07.761403",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:14:47.679628",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty; sticking",
"corrected_text": "at twenty: sticking",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \u201ctwenty\u201d from a semicolon to a colon (\"twenty;\" \u2192 \"twenty:\"). The prediction makes the same core change within a longer context string (\"at twenty; sticking\" \u2192 \"at twenty: sticking\"). Including extra surrounding words is acceptable, and the essential intended correction (semicolon \u2192 colon after \u201ctwenty\u201d) is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid it",
"corrected_text": "now, I have paid it",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The predicted edit also inserts a comma after \"now\" (\"now I have paid it\" \u2192 \"now, I have paid it\"). Although the prediction includes additional surrounding words, it captures the same core punctuation change in the same location. Therefore, it matches the ground truth's intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both identify the same original text (\"authorize\") and the same corrected text (\"authorise\"). The core change\u2014American spelling to British spelling\u2014is captured identically. Therefore the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam; \"not",
"corrected_text": "Clennam, \"not",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "him than",
"corrected_text": "him, than",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to break",
"corrected_text": "now, to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "*benevolence bubbling out of him*",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:56.847793",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:10:15.769323",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same corrected text (\"For, Mr. Pancks\"), capturing the core intended change: inserting a comma after \"For\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"quickly that\") and the same correction (\"quickly, that\"), inserting a comma after \"quickly\". The core intended punctuation change is captured exactly."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed-room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me, not",
"corrected_text": "making me; not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:57.993851",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:12:28.510152",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same corrected text (\"For, Mr. Pancks\"), capturing the core change of inserting a comma after \"For\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"quickly that\") and the same corrected text (\"quickly, that\"), capturing the identical core change\u2014adding a comma after \"quickly\". Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed-room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me, not",
"corrected_text": "making me; not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_claude_fable_5_1",
"date": "2026-09-06T04:16:54.915747",
"annotator": "Claude Fable 5.1 (OR)",
"annotation_date": "2026-09-06T04:14:34.029059",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"For\" in \"For Mr. Pancks\" to produce \"For, Mr. Pancks\". The core punctuation change is identical. Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"quickly that\") and the same corrected text (\"quickly, that\"), capturing the identical core change\u2014adding a comma after \"quickly\". Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed-room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2026-09-01",
"release_source": "https://www.anthropic.com/claude/fable",
"release_notes": "Successor to Claude Fable 5. Benchmark configuration uses high reasoning effort.",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Haiku 4.5",
"model_id": "claude-haiku-4.5",
"date": "2026-03-08T11:24:08.442225",
"shots": 2,
"config": {
"model_id": "claude-haiku-4.5",
"display_name": "Claude Haiku 4.5",
"shots": 2,
"temperature": 0.0,
"date": "2026-03-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:51:25.420270",
"file_results": [
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:24:34.790759",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:51:25.420270",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day,",
"corrected_text": "yonder, to-day,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2. Text Content Accuracy: The ground truth change inserts a comma after \"yonder\" (\"yonder to-day\" \u2192 \"yonder, to-day\"). The prediction captures the same essential change by converting \"yonder to-day,\" to \"yonder, to-day,\"\u2014i.e., adding the comma after \"yonder.\" The extra trailing comma is present in both the predicted original and corrected text, so it does not alter the core edit being made. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same correction (\"pockets, and\"), capturing the identical core change: inserting a comma after \"pockets\". Line numbers are ignored as instructed, so the discrepancy there does not affect correctness."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:13:19.697139",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:35:31.410672",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day,",
"corrected_text": "yonder, to-day,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"yonder\" (\"yonder to-day\" \u2192 \"yonder, to-day\"). The predicted change also inserts a comma after \"yonder\" (\"yonder to-day,\" \u2192 \"yonder, to-day,\"). Although the prediction includes an additional trailing comma in both the original and corrected text (extra surrounding context), the core intended punctuation correction\u2014adding the comma after \"yonder\"\u2014is captured correctly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth edit type.\n\nText Content Accuracy: The ground truth changes \"pockets and\" to \"pockets, and\" by inserting a comma after \"pockets.\" The predicted edit makes the identical change with the same original and corrected text. This captures the core intended correction precisely.\n\nLine numbers are ignored as instructed, and there are no other discrepancies."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:13:48.616953",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:54:05.071447",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction change \"yonder to-day\" to \"yonder, to-day\" by inserting a comma after \"yonder.\" The core intended punctuation change is captured exactly with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to make it \"pockets, and\". The original and corrected texts match the ground truth\u2019s intended punctuation change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:25:03.359476",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:51:58.369211",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "are\u2014\"He was going",
"corrected_text": "are\u2014\"He was going",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "stones?\" She",
"corrected_text": "stones?\" She",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"why did he dine to",
"corrected_text": "\"why did he dine to",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:14:14.846536",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:36:04.526233",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "are\u2014\"He was going",
"corrected_text": "are\u2014\"He was going",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "stones?\" She",
"corrected_text": "stones?\" She",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"why did he dine to",
"corrected_text": "\"why did he dine to",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:14:42.635638",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:54:37.962303",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "are\u2014\"He was going",
"corrected_text": "are\u2014\"He was going",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "stones?\" She",
"corrected_text": "stones?\" She",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"why did he dine to",
"corrected_text": "\"why did he dine to",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:11:31.391075",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:52:32.145546",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "first",
"corrected_text": "First",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 47,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:15:04.923293",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:36:36.682925",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "first",
"corrected_text": "First",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 47,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:15:29.227216",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:55:09.739401",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 48,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:25:30.722997",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:53:06.302510",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said (comma)",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam in amazement",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "far straiter and narrower than the broad high-road paved with vain pro-",
"corrected_text": "far straiter and narrower than the broad high-road paved with vain pro-",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:16:00.343424",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:37:10.751800",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said (comma)",
"corrected_text": "said,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "high-road paved",
"corrected_text": "high-road high-road paved",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said,when",
"corrected_text": "said, when",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:16:24.690768",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:55:44.920757",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said,",
"corrected_text": "said",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said (comma)",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "high-road paved",
"corrected_text": "high-road high-road paved",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:26:00.969193",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:53:39.971248",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Sun and Shadow.",
"corrected_text": "Sun and Shadow, Perioded here?",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:16:48.837245",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:37:41.758381",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Sun and Shadow.",
"corrected_text": "Sun and Shadow, Peried here?",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:17:14.554982",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:56:16.886185",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Sun and Shadow.",
"corrected_text": "Sun and Shadow, Perioded here?",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:12:30.673151",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:54:13.399252",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Chalons lay a long",
"corrected_text": "Chalons lay a long hyphen",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with wet",
"corrected_text": "sodden with wet",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:17:38.057834",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:38:28.010529",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Chalons lay a long",
"corrected_text": "Chalons lay a long hyphen",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "sodden with",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:18:01.183187",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:56:49.604894",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Chalons lay a long",
"corrected_text": "Chalons lay a long hyphen",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with wet",
"corrected_text": "sodden with wet",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:12:56.078846",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:54:47.735488",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart",
"corrected_text": "indeed thanked him from her heart",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "I hoped so",
"corrected_text": "I hoped so",
"observed_line_number": 47,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:18:23.776644",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:39:00.534484",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart",
"corrected_text": "indeed thanked him from her heart",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "I hoped so",
"corrected_text": "I hoped so",
"observed_line_number": 48,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:18:48.958947",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:57:25.502107",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart",
"corrected_text": "indeed thanked him from her heart",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "I hoped so",
"corrected_text": "I hoped so",
"observed_line_number": 47,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:26:29.019740",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:55:15.643602",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers\". Any line-number difference is to be ignored, so the intended edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 18,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change the same original string \"hers matters\" to the same corrected string \"hers, matters\", inserting a comma after \"hers\". This captures the core intended punctuation correction with no discrepancy in the text content.\n\nLine numbers are ignored as instructed, so the difference in line_number does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "abandoned the",
"corrected_text": "abandoned the",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly, \"they",
"corrected_text": "directly, \"They",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they",
"corrected_text": "\"They",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:19:17.155220",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:39:34.895103",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core punctuation change (inserting a comma after \"hers\"). Ignoring line numbers as instructed, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 18,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction apply the same core change: inserting a comma after \"hers\" to change \"hers matters\" into \"hers, matters.\" The original and corrected text strings match exactly, so the essential edit is correctly captured.\n\nLine numbers are ignored as instructed, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "peace\u2014or",
"corrected_text": "peace\u2014or",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "which\u2014that",
"corrected_text": "which\u2014that",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "abandoned",
"corrected_text": "abandoned",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "married.\"",
"corrected_text": "married.\"",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:19:46.380384",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:57:58.976344",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"hers and\" \u2192 \"hers, and\", inserting a comma after \"hers\". This captures the core intended change with no discrepancy in the edited text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 18,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the core intended change: insertion of a comma after \"hers\". Any discrepancy in line number is to be ignored per instructions.\n\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "peace\u2014or",
"corrected_text": "peace\u2014or",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "which\u2014that",
"corrected_text": "which\u2014that",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "they are to be married",
"corrected_text": "They are to be married",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ago, because",
"corrected_text": "ago, because",
"observed_line_number": 50,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:27:04.512573",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:55:55.784484",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once were",
"corrected_text": "that once were",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Thank you,",
"corrected_text": "Thank you",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore, and",
"corrected_text": "shore, and",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:20:20.379601",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:40:08.518713",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "cheerful",
"corrected_text": "cheerful",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once were",
"corrected_text": "that once were",
"observed_line_number": 42,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Thank you,",
"corrected_text": "Thank you",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore, and",
"corrected_text": "shore, and",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:20:53.711093",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:58:26.086858",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore, and",
"corrected_text": "shore, and",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "cheerful",
"corrected_text": "cheerful",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "that once",
"corrected_text": "that once",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Thank you,",
"corrected_text": "Thank you",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:27:53.235946",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:56:32.177454",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Chapter xxix .",
"corrected_text": "Chapter xxix",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mrs. Flintwinch goes on Dreaming ,",
"corrected_text": "Mrs. Flintwinch goes on Dreaming",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "THE house",
"corrected_text": "THE house c (ccp)",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "liege",
"corrected_text": "liege",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "luctant return of",
"corrected_text": "luctant return of",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,at",
"corrected_text": "existence, at",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it; to",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless,when",
"corrected_text": "motionless, when",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still; to",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view, by",
"corrected_text": "view, by",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence; is",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:21:43.755527",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:40:47.014408",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Chapter xxix .",
"corrected_text": "Chapter xxix",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mrs. Flintwinch goes on Dreaming ,",
"corrected_text": "Mrs. Flintwinch goes on Dreaming",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "THE house",
"corrected_text": "THE house c (ccp)",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "lucant return",
"corrected_text": "lucent return",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "still; to",
"corrected_text": "still; to",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "to",
"corrected_text": "to <",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,at",
"corrected_text": "existence, at",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it; to",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless,when",
"corrected_text": "motionless, when",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,by",
"corrected_text": "view, by",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence; is",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "liege",
"corrected_text": "liege",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:22:30.530646",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:59:02.578554",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Chapter xxix .",
"corrected_text": "Chapter xxix",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mrs. Flintwinch goes on Dreaming ,",
"corrected_text": "Mrs. Flintwinch goes on Dreaming",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "THE house",
"corrected_text": "THE house c (cap)",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "still; to",
"corrected_text": "still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "to lucutant return of",
"corrected_text": "lucutant return of",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,at",
"corrected_text": "existence, at",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view, by",
"corrected_text": "view, by",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence; is",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "liege",
"corrected_text": "liege",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:28:25.250821",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:57:05.773668",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that boiling-over old Christian",
"corrected_text": "that boiling-over old Christian",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 47,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 54,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest of snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:22:57.383797",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:41:19.505963",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that boiling-over old Christian",
"corrected_text": "that boiling-over old Christian",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 47,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 54,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest of snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:23:24.765063",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:59:35.040250",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that boiling-over old Christian",
"corrected_text": "that boiling-over old Christian",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 48,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest of snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T10:14:55.502718",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T09:57:39.913771",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "never and a faded check too",
"corrected_text": "never and a faded check too",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:23:46.478162",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T10:41:52.458006",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "never and a faded check too",
"corrected_text": "never and a faded check too",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-haiku-4.5",
"date": "2026-03-08T11:24:08.442225",
"annotator": "Claude Haiku 4.5",
"annotation_date": "2026-03-08T11:00:08.523742",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "never and a faded check too",
"corrected_text": "never and a faded check too",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2025-10-15",
"release_source": "https://www.anthropic.com/claude/haiku",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Opus 4.1",
"model_id": "claude-opus-4.1",
"date": "2026-03-08T11:26:00.381073",
"shots": 2,
"config": {
"model_id": "claude-opus-4.1",
"display_name": "Claude Opus 4.1",
"shots": 2,
"temperature": 0.0,
"date": "2026-03-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:51:46.626614",
"file_results": [
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:24:37.559843",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:51:46.626614",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same corrected text (\"pockets, and\"), capturing the identical core change\u2014adding a comma after \"pockets\". No discrepancies in the intended modification are present."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "Fellow Travellers",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day, sir",
"corrected_text": "to-day, sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:13:21.678679",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:35:52.405749",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\nText Content Accuracy: Both specify the same original phrase (\"pockets and\") and the same correction (\"pockets, and\"), inserting a comma after \"pockets.\" This captures the core intended change precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "Fellow Travellers",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day, sir",
"corrected_text": "to-day, sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:13:53.880064",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:54:35.797777",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"pockets and\" to \"pockets, and\", inserting a comma after \"pockets\". The core punctuation change is identical in the predicted edit and matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "Fellow Travellers",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day, sir",
"corrected_text": "to-day, sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:25:04.441218",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:52:26.132280",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "he might blame her father, if he saw them; that he might think,",
"corrected_text": "he might blame her father, if he saw them; that he might think, as a scar",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Why did he dine today, and leave this little creature to the mercy of the cold stones?\"",
"corrected_text": "\"Why did he dine today, and leave this little creature to the mercy of the cold stones?\" a reaction made",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:14:24.322640",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:36:31.981268",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "he might blame her father, if he saw them; that he might think,",
"corrected_text": "he might blame her father, if he saw them; that he might think, as a scar",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Why did he dine today, and leave this little creature to the mercy of the cold stones?\"",
"corrected_text": "\"Why did he dine today, and leave this little creature to the mercy of the cold stones?\" a reaction made",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:14:53.934077",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:55:14.098953",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "he might blame her father, if he saw them; that he might think,",
"corrected_text": "he might blame her father, if he saw them; that he might think, as a scar",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Why did he dine today, and leave this little creature to the mercy of the cold stones?\"",
"corrected_text": "\"Why did he dine today, and leave this little creature to the mercy of the cold stones?\" a reaction made",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:25:37.358864",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:53:03.920564",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 41,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction label the change as a \"replacement\", so the edit type matches.\n\n2. Text Content Accuracy: The ground truth changes \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the core intended substitution by replacing \"said\" with \"asked\". Although it omits the surrounding word \"Clennam\" from the original/corrected text fields, it still represents the essential correction accurately."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "do?",
"corrected_text": "do?",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "If",
"corrected_text": "If",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:15:29.568012",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:37:08.758806",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 41,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the verb \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam.\" The prediction captures the core intended change (\"said\" \u2192 \"asked\"), even though it omits the surrounding context \"Clennam.\" Since additional context is optional and the essential substitution is correctly identified, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "do?",
"corrected_text": "do?",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "If he",
"corrected_text": "cop If he",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:16:00.665799",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:55:54.524149",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 40,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam.\" The prediction replaces \"said\" \u2192 \"asked\" without including the trailing context (\"Clennam\"). Since the core intended change is the word substitution \"said\" to \"asked,\" and the prediction captures that essential modification, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "do?",
"corrected_text": "do?",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "If he",
"corrected_text": "If he",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:26:19.697779",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:53:49.092625",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change\u2014adding a comma after \"materials\". Line number differences are ignored per instructions. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 19,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same change to insert a comma after \"earnestly\", converting \"earnestly telling\" to \"earnestly, telling\". The core punctuation change is identical. Line number differences are to be ignored, so this does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"said when\") and the same correction (\"said, when\"), capturing the identical core punctuation change (inserting a comma after \"said\"). Line numbers are to be ignored, so the discrepancy there does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 28,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same core correction: changing \"said\" to \"said,\" by adding a comma. The essential edit is identical, and line numbers are to be ignored.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 32,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1. Edit Type Accuracy: Both the ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: Both edits change the same phrase from \"Clennam in\" to \"Clennam, in\", inserting a comma after \"Clennam\". This captures the core intended punctuation correction exactly. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "well but",
"corrected_text": "well; but",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "her for",
"corrected_text": "her, for",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "and said",
"corrected_text": "and said, (comma)",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir couldn't",
"corrected_text": "sir, couldn't",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "far straiter and narrower than the broad",
"corrected_text": "far straiter and narrower than the broad high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:16:37.040738",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:37:52.203122",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the core intended change\u2014adding a comma after \"materials\". Therefore, the essential edit is accurately represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), inserting a comma after \"earnestly.\" The core change is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"said\" in the phrase \"said when\" \u2192 \"said, when\". The original and corrected text segments match the ground truth\u2019s intended punctuation change. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 32,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"Clennam in\") and the same correction (\"Clennam, in\"), capturing the core intended change of inserting a comma after \"Clennam\". Ignoring line numbers as instructed, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "well but",
"corrected_text": "well; but",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "her for",
"corrected_text": "her, for",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "what in",
"corrected_text": "what, in",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "round and",
"corrected_text": "round, and",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir couldn't",
"corrected_text": "sir, couldn't",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "far straiter and narrower than the broad",
"corrected_text": "far straiter and narrower than the broad high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "and said",
"corrected_text": "and said, (comma)",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:17:17.408482",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:56:39.679669",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing\", inserting a comma after \"materials\". This captures the core intended punctuation correction with identical original and corrected text. (Line numbers are ignored as instructed.)"
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 19,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change: inserting a comma after \"earnestly\". Therefore, the prediction matches the ground truth's intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both specify the same core punctuation change: inserting a comma after \"said\" (\"said when\" \u2192 \"said, when\"). The original and corrected text match the ground truth\u2019s intended edit. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 32,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Clennam in\" to \"Clennam, in\" by inserting a comma after \"Clennam\". The core intended punctuation change is captured identically. Line number differences are to be ignored."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "well but",
"corrected_text": "well; but",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "her for",
"corrected_text": "her, for",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "what in",
"corrected_text": "what, in",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "round and",
"corrected_text": "round, and",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir couldn't",
"corrected_text": "sir, couldn't",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "far straiter and narrower than the broad",
"corrected_text": "far straiter and narrower than the broad high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "and said",
"corrected_text": "and said, (comma)",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:26:46.186369",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:54:24.988968",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Chapter I.",
"corrected_text": "Chapter I",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Sun and Shadow.",
"corrected_text": "Sun and Shadow",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:17:44.896717",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:38:31.850871",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Chapter I.",
"corrected_text": "Chapter I",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Sun and Shadow.",
"corrected_text": "Sun and Shadow",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:18:14.523249",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:57:15.929080",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Chapter I.",
"corrected_text": "Chapter I",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Sun and Shadow.",
"corrected_text": "Sun and Shadow",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:27:23.795113",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:55:07.485280",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons lay a long",
"corrected_text": "Chalons lay a long,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late, dull autumn night was closing in upon the river Sa\u00f4ne.",
"corrected_text": "A late, dull autumn night was closing in upon the river Sa\u00f4ne.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "heavy streak, occasionally made a little ragged by a row of poplar trees",
"corrected_text": "heavy streak, occasionally made a little ragged by a row of poplar-trees, hyphen",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "One man slowly moving on towards Chalons was the only visible",
"corrected_text": "One man, slowly moving on towards Chalons, was the only visible",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with wet;",
"corrected_text": "sodden with wet,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:18:53.462082",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:39:14.015490",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Chalons",
"corrected_text": "Ch\u00e2lons",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "lay a long heavy streak, occasionally made a little ragged by a row of poplar-trees",
"corrected_text": "lay a long, hyphen heavy streak, occasionally made a little ragged by a row of poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late, dull autumn night was closing in upon the river Sa\u00f4ne.",
"corrected_text": "A late, dull autumn night was closing in upon the river Sa\u00f4ne.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "One man slowly moving on towards Chalons was the only visible",
"corrected_text": "One man, slowly moving on towards Ch\u00e2lons, was the only visible",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Chalons",
"corrected_text": "Ch\u00e2lons",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with wet;",
"corrected_text": "sodden, with wet;",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:19:35.262358",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:57:58.713518",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Chalons",
"corrected_text": "Ch\u00e2lons",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late, dull autumn night was closing in upon the river Sa\u00f4ne.",
"corrected_text": "A late, dull autumn night was closing in upon the river Sa\u00f4ne.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "lay a long heavy streak, occasionally made a little ragged by a row of poplar-trees",
"corrected_text": "lay a long, heavy streak, occasionally made a little ragged by a row of poplar-trees, hyphen",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "One man slowly moving on towards Chalons",
"corrected_text": "One man, slowly moving on towards Chalons,",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Chalons",
"corrected_text": "Ch\u00e2lons",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with wet;",
"corrected_text": "sodden with wet;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:27:57.258994",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:55:57.097180",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "heart so much to its pain and trouble;",
"corrected_text": "heart, so much to its pain and trouble;",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so this criterion is satisfied.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"heart\" and \"so\" (\"heart so\" \u2192 \"heart, so\"). The prediction makes the same core change within a longer span of surrounding text (\"heart so much...\" \u2192 \"heart, so much...\"). Including extra context is acceptable, and the essential punctuation correction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "at length said Clennam,",
"corrected_text": "at length, said Clennam,",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction applies the same core punctuation change in a larger span of text (\"at length said Clennam,\" \u2192 \"at length, said Clennam,\"). Although it includes additional surrounding words, it correctly captures the essential intended edit: adding a comma after \"length\" before \"said.\""
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "tially up into his face, with some hurried words to the effect that she",
"corrected_text": "tially up into his face, with some hurried words to the effect that she indeed",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness), she gradually composed herself,",
"corrected_text": "ness), she gradually composed herself",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "thanked him from her heart (as she did,",
"corrected_text": "thanked him from her heart (as, she did,",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"That's well! I hope so;",
"corrected_text": "\"That's well! I hope so, I hope so;",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:20:13.004337",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:39:59.837970",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "heart so much to its pain and trouble;",
"corrected_text": "heart, so much to its pain and trouble;",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth indicates inserting a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The prediction applies the same core change within a longer span of surrounding text (\"heart so much to its pain and trouble;\" \u2192 \"heart, so much to its pain and trouble;\"). This correctly captures the essential punctuation correction (adding a comma after \"heart\")."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "at length said Clennam,",
"corrected_text": "at length, said Clennam,",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates inserting a comma after the word \"length\" (\"length said\" \u2192 \"length, said\"). The prediction applies the same core punctuation change in a longer surrounding context (\"at length said Clennam,\" \u2192 \"at length, said Clennam,\"). Although it includes extra words, the essential edit\u2014adding a comma after \"length\" before \"said\"\u2014is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "tially up into his face, with some hurried words to the effect that she",
"corrected_text": "tially up into his face, with some hurried words to the effect that she indeed",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness), she gradually composed herself,",
"corrected_text": "ness), she gradually composed herself",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "thanked him from her heart (as she did,",
"corrected_text": "thanked him from her heart (as, she did,",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"That's well! I hope so;",
"corrected_text": "\"That's well! I hope so, I hope so;",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:20:51.394975",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:58:37.910933",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "heart so much to its pain and trouble;",
"corrected_text": "heart, so much to its pain and trouble;",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The predicted edit also inserts a comma after \"heart\" within a longer phrase (\"heart so much to its pain and trouble;\" \u2192 \"heart, so much to its pain and trouble;\"). This captures the same core punctuation correction, just with additional surrounding context, which is acceptable."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "at length said Clennam,",
"corrected_text": "at length, said Clennam,",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma between \u201clength\u201d and \u201csaid\u201d (\"length said\" \u2192 \"length, said\"). The prediction performs the same core change, adding a comma after \u201clength\u201d in a longer surrounding phrase (\"at length said Clennam,\" \u2192 \"at length, said Clennam,\"). The extra context does not alter the essential correction, and the intended punctuation insertion is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "tially up into his face, with some hurried words to the effect that she",
"corrected_text": "tially up into his face, with some hurried words to the effect that she indeed",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness), she gradually composed herself,",
"corrected_text": "ness), she gradually composed herself",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "thanked him from her heart (as she did,",
"corrected_text": "thanked him from her heart (as, she did,",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"That's well! I hope so;",
"corrected_text": "\"That's well! I hope so, I hope so;",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:28:29.695046",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:56:38.270683",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"hers matters\" to \"hers, matters\", inserting a comma after \"hers\". The core intended punctuation correction is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change\u2014adding a comma after \"moment\". Line numbers are ignored as instructed, so this does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"beautiful\" in the phrase \"beautiful that\" \u2192 \"beautiful, that\". The original and corrected text strings match the ground truth\u2019s intention exactly. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "vigorous",
"corrected_text": "vigorous",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly \"they",
"corrected_text": "directly, \"They",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they",
"corrected_text": "\"They",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said hesitating",
"corrected_text": "said, hesitating",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:21:24.357088",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:40:42.474087",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same correction (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction make the same core change: inserting a comma after \"hers\" in the phrase \"hers matters,\" resulting in \"hers, matters.\" The original and corrected text strings match exactly, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"moment with\" to \"moment, with\", inserting a comma after \"moment\". This captures the core intended correction precisely (line number differences are ignored as instructed)."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction change \"beautiful that\" to \"beautiful, that,\" inserting a comma after \"beautiful.\" This captures the core intended punctuation correction with identical original and corrected text content.\n\nLine numbers are ignored as instructed, so the mismatch there does not affect correctness."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "vigorous",
"corrected_text": "vigorous",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly \"they",
"corrected_text": "directly, \"They",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they",
"corrected_text": "\"They",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said hesitating",
"corrected_text": "said, hesitating",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:21:53.916763",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:59:17.348414",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"hers\" in \"hers matters\" to produce \"hers, matters\". The core punctuation correction is identical.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"moment with\" to \"moment, with\" by inserting a comma after \"moment\". The original and corrected text strings match the ground truth\u2019s core intended change exactly (ignoring line numbers as instructed)."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: Both specify the same core change: inserting a comma after \"beautiful\" in the phrase \"beautiful that\", yielding \"beautiful, that\". The original_text and corrected_text match the ground truth\u2019s intended punctuation change.\n\nLine numbers are ignored as instructed, so the difference in line_number does not affect correctness."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "vigorous",
"corrected_text": "vigorous",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly \"they",
"corrected_text": "directly, \"They",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they",
"corrected_text": "\"They",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:29:01.231015",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:57:17.768813",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change \"shore and\" to \"shore, and\", inserting a comma after \"shore\". The core punctuation change is identical in original and corrected text.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "\"Thank you,\"",
"corrected_text": "",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:22:25.858157",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:41:31.717234",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"shore and\" to \"shore, and\", inserting a comma after \"shore\". The core punctuation change is identical in original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "\"Thank you,\"",
"corrected_text": "",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:22:54.024864",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:59:54.126372",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change of inserting a comma after \"shore\". Line numbers are ignored as instructed, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you,",
"corrected_text": "Thank you",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:30:02.591349",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:58:06.433302",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at the hour",
"corrected_text": "existence, at the hour",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change within a longer span of surrounding text (\"existence at the hour\" \u2192 \"existence, at the hour\"), which correctly places the comma immediately after \"existence\". The additional context does not alter the essential correction.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "motionless when we were",
"corrected_text": "motionless, when we were",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are labeled as a \"punctuation\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit adds the same comma after \"motionless\" within a longer span of surrounding text (\"motionless when we were\" \u2192 \"motionless, when we were\"). This includes extra context but preserves the core intended punctuation change. Therefore, the text content accurately captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "view by any larger",
"corrected_text": "view, by any larger",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core punctuation change, adding a comma immediately after \"view\" within a longer span (\"view by any larger\" \u2192 \"view, by any larger\"). The extra surrounding words are acceptable context and do not alter the essential intended edit.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "existence is the infirmity",
"corrected_text": "existence; is the infirmity",
"observed_line_number": 19,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation,\" and the predicted edit type is also \"punctuation,\" so the type matches.\n\n2) Text Content Accuracy: The ground truth changes the punctuation immediately after the word \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction applies the same core change in context by placing a semicolon right after \"existence\" (\"existence is the infirmity\" \u2192 \"existence; is the infirmity\"). Although the prediction\u2019s original span omits the comma explicitly, it still captures the essential intended correction: a semicolon should appear after \"existence\" at that point in the sentence."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to be unable",
"corrected_text": "stand-still, to be unable",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord and her",
"corrected_text": "liege, lord and her",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "The house in the city",
"corrected_text": "The house in the clear city",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring with its accompanying monotony,",
"corrected_text": "to each recurring with its accompanying monotony,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:23:50.264800",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:42:17.291219",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at the hour",
"corrected_text": "existence, at the hour",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change within a longer context span (\"existence at the hour\" \u2192 \"existence, at the hour\"), which still places the comma immediately after \"existence\". Including additional surrounding words is acceptable and does not alter the essential correction.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "motionless when we were",
"corrected_text": "motionless, when we were",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change by inserting a comma immediately after \"motionless\" within a longer span (\"motionless when we were\" \u2192 \"motionless, when we were\"). Including extra surrounding context is acceptable, and the essential punctuation correction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "view by any larger",
"corrected_text": "view, by any larger",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction adds the same comma after \"view\" within a longer phrase (\"view by any larger\" \u2192 \"view, by any larger\"). Although the prediction includes extra surrounding words, it correctly captures the core intended punctuation change: inserting a comma after \"view\"."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to be unable",
"corrected_text": "stand-still, to be unable",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord and her",
"corrected_text": "liege, lord and her",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "The house in the city",
"corrected_text": "The house in the clear city",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring with its accompanying monotony,",
"corrected_text": "to each recurring with its accompanying monotony,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is the infirmity",
"corrected_text": "existence; is the infirmity",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:24:27.510619",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T11:00:40.177002",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction expands the span to include the following word (\"existence at\" \u2192 \"existence, at\"), but it still captures the same core correction: insertion of a comma immediately after \"existence.\" The added context does not alter the essential edit intent."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit changes \"motionless when\" \u2192 \"motionless, when\", which likewise inserts a comma immediately after \"motionless\". The extra surrounding word \"when\" is acceptable context and does not alter the core intended correction.\n\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds the same comma after \"view\" but includes extra surrounding context (\"view by\" \u2192 \"view, by\"). This still captures the essential intended correction: inserting a comma immediately after \"view\"."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it, to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege, lord",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "The house in the city",
"corrected_text": "The house in the clear city",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring with its accompanying monotony, always the same reluctant return",
"corrected_text": "each recurring with its accompanying monotony, always the same reluctant return to",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:30:30.074085",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:58:46.462298",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "italicize",
"corrected_text": "italicize",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "authorise",
"corrected_text": "authorise",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "driest of snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bubbling out of him",
"corrected_text": "bubbling out of him",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:24:43.460175",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:42:55.000223",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": 7,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"of the dryest snorts.\") and the same corrected text (\"of the driest of snorts.\"), capturing the identical core change (spelling correction dryest\u2192driest and insertion of \"of\"). Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Italicize",
"corrected_text": "Italicize",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "authorise",
"corrected_text": "authorise",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:25:09.277564",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T11:01:18.861592",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "boiling-over",
"corrected_text": "the boiling-over",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Italicize",
"corrected_text": "Italicize",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 44,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "driest of snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T10:30:55.100098",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T09:59:37.387917",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"For\" in the phrase \"For Mr. Pancks\" to make it \"For, Mr. Pancks\". The original_text and corrected_text match the ground truth exactly, capturing the core intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". The core punctuation change is identical.\n\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam of course",
"corrected_text": "Clennam, of course,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time but",
"corrected_text": "time; but",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "never and",
"corrected_text": "never, and",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur for",
"corrected_text": "Arthur, for",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "visit but",
"corrected_text": "visit; but",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur returned",
"corrected_text": "Arthur,\" returned",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Finching pray",
"corrected_text": "Finching, \"pray",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "still tis",
"corrected_text": "still 'tis",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and if",
"corrected_text": "and, if",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view but",
"corrected_text": "view, but",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time but",
"corrected_text": "time, but",
"observed_line_number": 44,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Cucumber and",
"corrected_text": "Cucumber, and",
"observed_line_number": 47,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:25:35.459381",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T10:43:41.931551",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the edit type matches exactly.\n2. Text Content Accuracy: Both edits change \"For Mr. Pancks\" to \"For, Mr. Pancks\", inserting a comma after \"For\". The core punctuation change is identical, with no discrepancies in original or corrected text. Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change, inserting a comma after \"quickly\" (\"quickly that\" \u2192 \"quickly, that\"). The core intended punctuation correction is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "morning and",
"corrected_text": "morning, and",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Rugg from",
"corrected_text": "Rugg, from",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "back and",
"corrected_text": "back, and",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam of course",
"corrected_text": "Clennam, of course,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time but",
"corrected_text": "time, but",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "word and",
"corrected_text": "word, and",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur for",
"corrected_text": "Arthur, for",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "visit but",
"corrected_text": "visit, but",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Finching pray",
"corrected_text": "Finching, pray",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that and",
"corrected_text": "that, and",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view but",
"corrected_text": "view, but",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time but",
"corrected_text": "time, but",
"observed_line_number": 44,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.1",
"date": "2026-03-08T11:26:00.381073",
"annotator": "Claude Opus 4.1",
"annotation_date": "2026-03-08T11:02:01.773145",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same corrected text (\"For, Mr. Pancks\"), capturing the identical core change\u2014adding a comma after \"For\". Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original span (\"quickly that\") and the same correction (inserting a comma: \"quickly, that\"). The core intended change\u2014adding a comma after \"quickly\"\u2014is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam of course",
"corrected_text": "Clennam, of course,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time but",
"corrected_text": "time; but",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "never and",
"corrected_text": "never, and",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur for",
"corrected_text": "Arthur, for",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Finching pray",
"corrected_text": "Finching, pray",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and if",
"corrected_text": "and, if",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view but",
"corrected_text": "view, but",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time but",
"corrected_text": "time, but",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2025-08-05",
"release_source": "https://www.anthropic.com/news/claude-opus-4-1",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Opus 4.5",
"model_id": "claude-opus-4.5",
"date": "2026-03-08T11:27:19.059572",
"shots": 2,
"config": {
"model_id": "claude-opus-4.5",
"display_name": "Claude Opus 4.5",
"shots": 2,
"temperature": 0.0,
"date": "2026-03-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:00:53.304193",
"file_results": [
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:24:37.028652",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:00:53.304193",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the core intended change of inserting a comma after \"howling\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same correction (inserting a comma after \"yonder\" to make \"yonder, to-day\"). The core change is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same corrected text (\"pockets, and\"), capturing the core intended change: inserting a comma after \"pockets\". Line numbers are ignored per instructions, so this does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:13:23.850141",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:44:43.747902",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both identify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change: inserting a comma after \"howling\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), inserting a comma after \"yonder\". The core change is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to make it \"pockets, and\". The original_text and corrected_text match the ground truth\u2019s intention. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:13:57.525569",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:03:03.376380",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"howling\" in the phrase \"howling over\", resulting in \"howling, over\". The original_text and corrected_text match the ground truth precisely, capturing the core intended punctuation change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the identical core change: insertion of a comma after \"yonder.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same corrected text (\"pockets, and\"), capturing the identical core change: inserting a comma after \"pockets.\" Line number differences are to be ignored, so the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:25:09.013124",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:01:36.026922",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "\"No sir",
"corrected_text": "\"No, sir",
"observed_line_number": 11,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"No\" and \"sir\" (\"No sir\" \u2192 \"No, sir\"). The predicted edit also inserts that same comma (\"\\\"No sir\" \u2192 \"\\\"No, sir\"). Although the prediction includes an additional leading quotation mark as extra context, it preserves the same core punctuation correction intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"why",
"corrected_text": "\"Why",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones!\"",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:14:30.739920",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:45:20.681092",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "\"No sir",
"corrected_text": "\"No, sir",
"observed_line_number": 11,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after \"No\" in \"No sir\" \u2192 \"No, sir.\" The prediction captures the same core punctuation change, adding the comma after \"No\" as well. Although the prediction includes an additional leading quotation mark as extra context (\"No sir \u2192 \"No, sir), this does not alter the essential intended correction (the comma insertion). Therefore, the text content aligns with the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "'why",
"corrected_text": "'Why",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones!'",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:15:03.931958",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:03:43.350958",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "\"No sir",
"corrected_text": "\"No, sir",
"observed_line_number": 11,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"No\" in \"No sir\" \u2192 \"No, sir\". The prediction performs the same core punctuation change, but includes an additional leading quotation mark as context: \"No sir \u2192 \"No, sir. This still captures the essential intended edit (comma insertion after \"No\").\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"why",
"corrected_text": "\"Why",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!\"",
"corrected_text": "stones?\"",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:25:41.548248",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:02:12.111770",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the type matches.\n\n2) Text Content Accuracy: The ground truth changes \"said Clennam\" to \"asked Clennam\", whose core correction is replacing the word \"said\" with \"asked\". The prediction captures this essential change by replacing \"said\" with \"asked\" (even though it includes less surrounding context). Since the key intended substitution is correctly represented, the text content matches the ground truth's intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "do? If he",
"corrected_text": "do? I can't help it. If he",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:15:32.393952",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:45:54.514134",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth intends to change the verb \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the core substitution \"said\" \u2192 \"asked\" and simply omits the surrounding context (\"Clennam\"), which remains unchanged in the ground truth. Since the essential change is correctly identified, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "do? If",
"corrected_text": "do? I can, if",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:16:06.276136",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:04:18.531133",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit, so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"said Clennam\" to \"asked Clennam\", whose core intended modification is the word substitution \"said\" \u2192 \"asked\". The predicted edit captures this essential change by replacing \"said\" with \"asked\" (with less surrounding context). Omitting \"Clennam\" does not alter the core correction being made, so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "do? If he",
"corrected_text": "do? I can't help it. If he",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:26:24.127908",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:02:52.410201",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"high road\") and the same corrected text (\"high-road\"), capturing the identical core change of inserting a hyphen. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing\" by inserting a comma after \"materials\". This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"about and\" \u2192 \"about, and\", inserting a comma after \"about\". This captures the core intended change with identical original and corrected text. Line numbers are to be ignored, so the discrepancy there does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: The ground truth changes \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly\". The prediction makes the exact same change from the same original text to the same corrected text. This captures the core intended punctuation correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 21,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same core punctuation change, inserting a comma after \"said\" in the phrase \"said when\" \u2192 \"said, when.\" This captures the essential intended correction with no discrepancy in the modified text.\n\nLine numbers are ignored as instructed, so the prediction correctly reflects the ground truth edit."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 28,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original_text (\"Clennam in\") and the same corrected_text (\"Clennam, in\"), capturing the identical core change: inserting a comma after \"Clennam\". Ignoring line numbers, the predicted edit fully matches the ground truth intent."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said, (comma)",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:16:45.968363",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:46:33.467013",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road,\" correctly capturing the core intended change (adding a hyphen). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing\" by inserting a comma after \"materials.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original_text (\"about and\") and the same corrected_text (\"about, and\"), capturing the identical core change\u2014adding a comma after \"about\". Line numbers are ignored as instructed. Therefore, the prediction matches the ground truth's intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly\". This captures the core intended punctuation change with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 21,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"said when\" and the same corrected text \"said, when\", capturing the identical core change (inserting a comma after \"said\"). Line number differences are ignored as instructed."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 28,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both ground truth and prediction change \"Clennam in\" to \"Clennam, in\", inserting a comma after \"Clennam\". The core punctuation change is identical, so the text content matches the ground truth intention. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said, (comma)",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:17:28.101573",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:04:58.683030",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"high road\" to \"high-road\", capturing the identical core correction (inserting a hyphen). No discrepancy in original or corrected text content."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing,\" inserting a comma after \"materials.\" The core punctuation change is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original phrase \"about and\" and the same correction \"about, and\", capturing the identical core punctuation change (inserting a comma after \"about\"). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction apply the same core change: inserting a comma after \"earnestly\" in the phrase \"earnestly telling\" to produce \"earnestly, telling.\" The original and corrected text strings match exactly, so the essential edit is correctly captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 21,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"said when\" \u2192 \"said, when\", inserting a comma after \"said\". This captures the core intended punctuation change with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 28,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same corrected text (\"Clennam, in\"), capturing the core intended change of inserting a comma after \"Clennam\". Line number differences are irrelevant per instructions."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said, (comma)",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:26:49.094965",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:03:28.416534",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:18:02.144546",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:47:06.497309",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:18:32.127249",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:05:39.220761",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:27:36.556014",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:04:08.481256",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change\u2014adding a period to \"Let Loose\" to make it \"Let Loose.\" The original and corrected text strings match the ground truth intention precisely.\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"dull autumn\") and the same corrected text (\"dull, autumn\"), capturing the identical core change\u2014adding a comma after \"dull\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"man slowly\" to \"man, slowly\", inserting a comma after \"man\". The core intended punctuation correction is captured identically. Therefore, the predicted edit matches the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change of inserting a comma after \"Chalons\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "sodden-ed with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:19:19.174282",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:47:44.724915",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have the type \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"Let Loose\" to \"Let Loose.\" (adding a period). The core punctuation correction is captured identically. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"Chalons was\" \u2192 \"Chalons, was\", i.e., inserting a comma after \"Chalons\". The core punctuation change is identical in the predicted edit and the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "sodden-ed with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:20:05.021389",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:06:34.756683",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change, adding a period: \"Let Loose\" \u2192 \"Let Loose.\". There is no discrepancy in original or corrected text content. Therefore the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Chalons was\" to \"Chalons, was\" by inserting a comma after \"Chalons\". The core punctuation change is identical, with no discrepancies in original or corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "sodden-ed with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:28:10.860090",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:04:49.645918",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"heart so\" to \"heart, so,\" correctly capturing the insertion of a comma after \"heart\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction does the same core punctuation insertion, but includes additional preceding context (\"at length said\" \u2192 \"at length, said\"). This captures the essential intended change (adding the comma after \"length\")."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 33,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change (present tense to past tense). Any difference in line number is to be ignored per instructions, so the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestness) she",
"corrected_text": "earnestness), she",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "(as she did if",
"corrected_text": "(as she did, if",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:20:48.491631",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:48:20.766192",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original segment (\"heart so\") and the same correction (\"heart, so\"), capturing the identical core change of inserting a comma after \"heart\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 33,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction performs the same core punctuation change, inserting a comma after \"length\" (\"at length said\" \u2192 \"at length, said\"). The extra surrounding context word \"at\" does not alter the essential correction. Therefore, the predicted edit captures the ground truth\u2019s intended change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 36,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"I hope so\" with \"I hoped so\". This captures the same core change (tense shift from present to past). Line numbers are ignored as instructed, and there is no discrepancy in the text change itself."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "(as she",
"corrected_text": "(as, she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestness)",
"corrected_text": "earnestness),",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (as she did, if it be the source of earnestness)",
"corrected_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (as indeed she did, if it be the source of earnestness),",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:21:24.623914",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:07:19.326521",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"heart so\") and the same corrected text (\"heart, so\"), capturing the identical core change\u2014adding a comma after \"heart\". Therefore, the prediction matches the ground truth\u2019s intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: inserting a comma so that \"length said\" becomes \"length, said.\" The original_text and corrected_text match the ground truth\u2019s intended punctuation change. Line number differences are to be ignored, so the prediction captures the ground truth edit correctly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 33,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the exact same phrase \"I hope so\" with \"I hoped so\". This captures the core intended change (tense shift from present to past) with no discrepancy in original or corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "(as'she",
"corrected_text": "(as she",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestness) she",
"corrected_text": "earnestness), she",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "tially",
"corrected_text": "indeed",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:28:46.220404",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:05:27.965754",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same core punctuation change, inserting a comma after \"hers\" in the phrase \"hers and\" to make it \"hers, and\". The original_text and corrected_text match the ground truth\u2019s intended change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change: inserting a comma after \"hers\". Line numbers are ignored as instructed. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change: insertion of a comma after \"moment\". Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same corrected text (\"beautiful, that\"), capturing the identical core change\u2014adding a comma after \"beautiful\". Line numbers are ignored as instructed, so the prediction matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they",
"corrected_text": "\"They",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:21:54.733672",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:48:57.927062",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change: insertion of a comma after \"hers\". Line numbers are ignored as instructed. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change the phrase from \"hers matters\" to \"hers, matters\", inserting a comma after \"hers\". The original_text and corrected_text match the ground truth\u2019s intended core change exactly.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change\u2014adding a comma after \"moment\". Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"beautiful that\" to \"beautiful, that,\" inserting a comma after \"beautiful.\" This captures the identical core correction. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:22:30.935857",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:08:02.518865",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers.\" Line numbers are to be ignored, so the difference there does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change: insertion of a comma after \"hers\". Any line-number discrepancy is to be ignored. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change: insertion of a comma after \"moment\". Line number differences are to be ignored, so the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same correction (\"beautiful, that\"), inserting a comma after \"beautiful\". This captures the core intended change precisely. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they",
"corrected_text": "\"They",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:29:20.613049",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:06:09.203508",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change, inserting a comma to change \"shore and\" to \"shore, and\". The original and corrected text content are identical between ground truth and prediction. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2. Text Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\". This captures the same core change intended by the ground truth, with no discrepancy in the actual substitution."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you\"",
"corrected_text": "you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the moonlight",
"corrected_text": "the moonlight",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "is now.\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:23:03.617181",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:49:36.074961",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change (inserting a comma after \"shore\"). Ignoring line numbers, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n\nText Content Accuracy: Both specify replacing the original text \"that\" with the corrected text \"than.\" This captures the same core intended correction. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you\"",
"corrected_text": "you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the moonlight",
"corrected_text": "the moonlight",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "is now.\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:23:36.604671",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:08:47.406166",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change, inserting a comma after \"shore\" in the phrase \"shore and\" to make it \"shore, and\". The original_text and corrected_text match the ground truth\u2019s intended edit. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"that\" with \"than\", capturing the same core correction. Ignoring line numbers as instructed, the predicted edit matches the ground truth\u2019s intended change."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you\"",
"corrected_text": "you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the moonlight",
"corrected_text": "the moonlight",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "is now.\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:29:58.640075",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:06:50.727845",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction also inserts a comma immediately after \"existence\" (\"existence at\" \u2192 \"existence, at\"). Although the prediction includes the following word \"at\" as additional context, it captures the same core punctuation change at the correct location.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change, adding a comma immediately after \"motionless\" in the larger phrase (\"motionless when\" \u2192 \"motionless, when\"). Including the extra surrounding word \"when\" is acceptable context, and the essential punctuation correction is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to add a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction adds the comma after \"view\" as well, shown with extra surrounding context (\"view by\" \u2192 \"view, by\"). This captures the same core punctuation insertion intended by the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "return of",
"corrected_text": "return, of",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with them;",
"corrected_text": "with them,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it, to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "the city",
"corrected_text": "the c(car) city",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:24:16.424531",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:50:19.372031",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core punctuation change within a slightly larger span of text (\"existence at\" \u2192 \"existence, at\"). This correctly captures the essential intention: inserting a comma immediately after \"existence\"."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction also inserts a comma immediately after \"motionless\" (\"motionless when\" \u2192 \"motionless, when\"). The prediction includes additional surrounding context (the following word \"when\"), but it captures the same core punctuation change intended by the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit also adds a comma immediately after \"view\" while including extra surrounding context (\"view by\" \u2192 \"view, by\"). This captures the same core punctuation change intended by the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "return of",
"corrected_text": "return, of",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with them;",
"corrected_text": "with them,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it, to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "the city",
"corrected_text": "the C(car) city",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:25:17.994649",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:09:33.052698",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 11,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both edits change the same token from \"them,\" to \"them;\". This captures the core intended punctuation correction precisely. Line numbers are ignored as instructed, and there is no discrepancy in the actual text change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change, inserting a comma immediately after \"existence\" within a larger context (\"existence at\" \u2192 \"existence, at\"). Including the following word \"at\" as context does not alter the essential edit. Therefore, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "it, to",
"corrected_text": "it; to",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"it,\" to \"it;\" (comma to semicolon after \"it\"). The predicted edit changes \"it, to\" to \"it; to,\" which includes additional surrounding context (the word \"to\") but preserves the same core punctuation change at the same location (comma after \"it\" replaced with a semicolon). Therefore, the prediction captures the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change, adding a comma immediately after \"motionless\" within a larger phrase (\"motionless when\" \u2192 \"motionless, when\"). Including the following word \"when\" is extra context but does not alter the essential punctuation edit. Therefore, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "stand-still, to",
"corrected_text": "stand-still; to",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The prediction performs the same core change, simply including additional following context (\"stand-still, to\" \u2192 \"stand-still; to\"). This preserves the essential intended correction (comma replaced by semicolon at the same location), so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core punctuation change by inserting a comma immediately after \"view\" within a larger context (\"view by\" \u2192 \"view, by\"). The added context does not alter the essential edit; it still captures the intended insertion of a comma after \"view\"."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "existence, is",
"corrected_text": "existence; is",
"observed_line_number": 19,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes the punctuation after \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction applies the same core punctuation change, simply including extra surrounding context (\"existence, is\" \u2192 \"existence; is\"). This preserves the essential intended correction.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "gity",
"corrected_text": "city",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return pf",
"corrected_text": "return of",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each",
"corrected_text": "to each",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liegelord",
"corrected_text": "liege lord",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:30:43.866062",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:07:28.737995",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts \"of\" so the phrase becomes \"driest of snorts\" (within the larger context \"of the ... snorts.\"). The predicted edit also changes \"dryest snorts\" to \"driest of snorts,\" capturing the same essential correction (spelling change plus insertion of \"of\"). Although the prediction omits surrounding context (\"of the\" and the final period), it still accurately captures the core intended change.\n\nTherefore, the predicted edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him,",
"corrected_text": "him,",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"italicize,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth indicates italicizing the word \"him.\" The prediction applies italicization to \"him,\" but includes an adjacent comma (\"him,\"). This is additional context rather than a different core change\u2014the essential intent (italicize \"him\") is still captured. Including the comma does not contradict the ground truth\u2019s intended italicization of the word itself."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as a \"replacement\", so the type matches exactly.\n\n2) Text Content Accuracy: Both indicate the same substitution of \"authorize\" with \"authorise\". The core change (American to British spelling) is identical in original and corrected text.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that-boiling-over",
"corrected_text": "that boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty; sticking",
"corrected_text": "twenty, sticking",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:25:52.694656",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:50:58.560167",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts \"of\" to make the phrase \"driest of snorts\" (within the larger context \"of the ... .\"). The predicted edit also changes \"dryest\" \u2192 \"driest\" and produces \"driest of snorts\", capturing the same core correction. The prediction omits surrounding context (\"of the\" and the final period), but those elements are not part of the actual change in the ground truth and are permissible to omit. Therefore, the prediction matches the ground truth\u2019s intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2. Text Content Accuracy: Both specify replacing \"authorize\" with \"authorise\". This captures the core intended correction (American to British spelling). Ignoring line numbers as instructed, the essential edit is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that, boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty:",
"corrected_text": "twenty;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Your own recompense",
"corrected_text": "Your own recompense",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "him",
"corrected_text": "him,",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:26:26.139957",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:10:17.002916",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"dryest snorts\" to \"driest of snorts\" within the larger phrase \"of the dryest snorts.\" \u2192 \"of the driest of snorts.\" The prediction captures the same core correction: correcting the spelling \"dryest\" \u2192 \"driest\" and inserting \"of\" before \"snorts\". The prediction omits surrounding context (\"of the\" and the final period), but those elements are not part of the intended change and are allowed to be excluded as long as the essential edit is preserved.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that, boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty:",
"corrected_text": "twenty;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him,",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "authorise",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T10:31:10.127654",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:08:05.581450",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction likewise inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the core intended punctuation change at the correct location."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that\", i.e., inserting a comma after \"quickly\". The core change is identical. Therefore the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me; not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:26:51.800298",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T10:51:33.444576",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"For Mr. Pancks\" to \"For, Mr. Pancks\", i.e., it inserts a comma after \"For\". The prediction changes \"For Mr.\" to \"For, Mr.\", which also inserts a comma after \"For\". Although the prediction includes less surrounding context (it stops at \"Mr.\" and omits \"Pancks\"), it still captures the core intended punctuation change at the correct location. Therefore, the essential edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original phrase \"quickly that\" and the same correction \"quickly, that\", inserting a comma after \"quickly\". The essential punctuation change is identical. Therefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me; not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.5",
"date": "2026-03-08T11:27:19.059572",
"annotator": "Claude Opus 4.5",
"annotation_date": "2026-03-08T11:10:57.924630",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The prediction captures the same core punctuation change by editing \"For Mr.\" \u2192 \"For, Mr.\". Although it omits the trailing context (\"Pancks\"), it still correctly represents the essential intended edit: adding a comma immediately after \"For\" before \"Mr.\"."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the identical change from \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". The core correction is captured exactly with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me; not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2025-11-01",
"release_display": "Nov 2025",
"release_source": "https://www.anthropic.com/transparency",
"release_notes": "Official Anthropic source used here lists November 2025 but not an exact day.",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Opus 4.6",
"model_id": "claude-opus-4.6",
"date": "2026-03-08T11:26:43.285854",
"shots": 2,
"config": {
"model_id": "claude-opus-4.6",
"display_name": "Claude Opus 4.6",
"shots": 2,
"temperature": 0.0,
"date": "2026-03-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:01:47.915199",
"file_results": [
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:24:42.349414",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:01:47.915199",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change the phrase from \"howling over\" to \"howling, over,\" inserting a comma after \"howling.\" The original and corrected text match exactly, capturing the core intended punctuation change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the identical core change\u2014adding a comma after \"yonder\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same corrected text (\"pockets, and\"), capturing the identical core punctuation change (inserting a comma after \"pockets\"). Line numbers are ignored per instructions, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "TRAVELLERS",
"corrected_text": "TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sir;",
"corrected_text": "sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:13:26.236030",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:45:39.761806",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"yonder to-day\" to \"yonder, to-day\", inserting a comma after \"yonder\". The original and corrected texts match exactly, capturing the core intended punctuation change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same correction (\"pockets, and\"), capturing the intended insertion of a comma after \"pockets\". Therefore the core change matches the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "TRAVELLERS",
"corrected_text": "TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sir;",
"corrected_text": "sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:14:01.548836",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:04:00.358344",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"howling over\" to \"howling, over,\" inserting a comma after \"howling.\" This captures the core intended punctuation change with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction apply the same core punctuation change, inserting a comma after \"yonder\" to change \"yonder to-day\" to \"yonder, to-day.\" There are no differences in original or corrected text that would alter the intended edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"pockets and\") and the same correction (\"pockets, and\"), capturing the identical core change of inserting a comma after \"pockets.\" Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "TRAVELLERS",
"corrected_text": "TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sir;",
"corrected_text": "sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:25:15.559069",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:02:26.583586",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "cold stones!\" She",
"corrected_text": "cold stones?\" She",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the types match exactly.\n\n2. Text Content Accuracy: The ground truth changes the punctuation in \"cold stones!\" to \"cold stones?\" (exclamation mark to question mark). The predicted edit changes \"cold stones!\\\" She\" to \"cold stones?\\\" She\", which includes extra surrounding context but preserves the same core change: replacing the exclamation mark after \"cold stones\" with a question mark. This captures the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"Why did he dine to-",
"corrected_text": "\"why did he dine to-",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No, sir, i have",
"corrected_text": "No, sir, I have",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:14:43.554053",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:46:16.784521",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "\"why did he dine to-",
"corrected_text": "\"Why did he dine to-",
"observed_line_number": 29,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the initial word \"why\" to \"Why\" in the phrase \"why did he dine\". The prediction likewise changes \"why\" to \"Why\" (\"\\\"why did he dine to-\" \u2192 \"\\\"Why did he dine to-\"). Although the prediction includes additional surrounding context (a leading quotation mark and the following \"to-\"), it captures the same core capitalization change identified in the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "cold stones!\" She",
"corrected_text": "cold stones?\" She",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The core change in the ground truth is replacing the exclamation mark with a question mark in \"cold stones!\" \u2192 \"cold stones?\". The prediction performs the same essential punctuation substitution, just with additional surrounding context (\"\\\" She\"). This extra context does not alter the intended edit, and the key change (! \u2192 ?) is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No, sir, i have",
"corrected_text": "No, sir, I have",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:15:23.384500",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:04:37.731041",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "\"why did he dine",
"corrected_text": "\"Why did he dine",
"observed_line_number": 29,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"capitalization\", which matches the ground truth exactly.\n\n2. Text Content Accuracy: The ground truth change is capitalizing the initial \"why\" \u2192 \"Why\" in the phrase \"why did he dine\". The prediction captures the same core change, but includes an opening quotation mark as additional surrounding context (\"\\\"why did he dine\" \u2192 \"\\\"Why did he dine\"). This extra context does not alter the essential intended correction (capitalizing the first word), so the text-content intention matches."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "cold stones!\" She",
"corrected_text": "cold stones?\" She",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes the punctuation in \"cold stones!\" to \"cold stones?\" (exclamation mark to question mark). The prediction makes the same core change within a slightly longer context: \"cold stones!\\\" She\" \u2192 \"cold stones?\\\" She\". Including extra surrounding characters (the closing quote and following word) does not alter the essential intent. The key punctuation change is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No, sir, i have",
"corrected_text": "No, sir, I have",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:25:44.489782",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:03:03.037732",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces the word \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam.\" The prediction captures the same core substitution \"said\" \u2192 \"asked\" even though it omits the surrounding context (\"Clennam\"). Since the essential intended change is the replacement of \"said\" with \"asked,\" the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "If",
"corrected_text": "I cap If",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:15:54.682578",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:46:51.950986",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are \"replacement\" edits, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"said\" to \"asked\" (within the phrase \"said Clennam\" \u2192 \"asked Clennam\"). The prediction captures the same core substitution \"said\" \u2192 \"asked\" but with less surrounding context. Since the essential change is identical and additional context is optional, the prediction matches the ground truth\u2019s intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "If",
"corrected_text": "I cap If",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:16:25.862934",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:05:16.433379",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the core change by replacing \"said\" \u2192 \"asked\". Although it omits the surrounding context word \"Clennam,\" it still represents the essential intended substitution identified in the ground truth.\n\nBecause both the edit type and the essential text change match the ground truth\u2019s intention, the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "If",
"corrected_text": "I cap If",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:26:34.105764",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:03:42.893307",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"materials costing\" to \"materials, costing\" by inserting a comma after \"materials\". The core punctuation change is identical. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"about\" in the phrase \"about and\" to make it \"about, and\". The core correction is identical. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"earnestly telling\" to \"earnestly, telling\", inserting a comma after \"earnestly\". This captures the core intended punctuation change with identical original and corrected text. Therefore, the text content matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"said when\") and the same corrected text (\"said, when\"), capturing the core intended change of inserting a comma after \"said\". Any line-number discrepancy is ignored per instructions.\n\nTherefore, the predicted edit correctly matches the ground truth edit\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"Clennam in\" to \"Clennam, in\", inserting a comma after \"Clennam\". The core change is identical. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said:",
"corrected_text": "and said, (comma)",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:17:08.752813",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:47:30.618594",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edit have the type set to \"punctuation\", so the edit type matches exactly.\n\n2. Text Content Accuracy: Both specify the same change from \"materials costing\" to \"materials, costing\", inserting a comma after \"materials\". The core punctuation correction is identical in original and corrected text.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma to change \"about and\" to \"about, and\". The original and corrected text strings align with the ground truth\u2019s intended punctuation correction.\nTherefore, the predicted edit correctly captures the ground truth intention (ignoring line numbers as instructed)."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change: inserting a comma after \"earnestly\". Any line-number discrepancy is to be ignored, so the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original segment \"said when\" and the same corrected segment \"said, when\", capturing the identical core change (inserting a comma after \"said\"). Any difference in line number is to be ignored per instructions, so the essential edit is correctly represented."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"Clennam in\" to \"Clennam, in\", inserting a comma after \"Clennam\". This captures the core intended punctuation change with identical original and corrected text."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said:",
"corrected_text": "and said, (comma)",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:17:51.659380",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:06:02.545475",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing\" by inserting a comma after \"materials.\" The core punctuation change is identical, so the intended correction is accurately captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both specify the same core change: inserting a comma to change \"about and\" to \"about, and\". The original and corrected text strings match the ground truth\u2019s intended correction. Line number differences are to be ignored, so this does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change: insertion of a comma after \"earnestly\". Therefore, the intended edit is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits apply the same core correction\u2014adding a comma after \"said\" in the phrase \"said when\" to produce \"said, when\". The original and corrected text strings match the ground truth\u2019s intended change. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both edits change \"Clennam in\" to \"Clennam, in\" by inserting a comma after \"Clennam\". The original and corrected text match the ground truth exactly, capturing the core punctuation change."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said:",
"corrected_text": "and said, (comma)",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:12:44.900459",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:04:17.761153",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Shadow.",
"corrected_text": "Shadow. Period here?",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:18:16.628062",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:48:05.864223",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Shadow.",
"corrected_text": "Shadow. Period here?",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:18:39.873291",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:06:50.479004",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Shadow.",
"corrected_text": "Shadow. Period here?",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:27:22.620901",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:04:56.891955",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The core intended punctuation change is captured exactly with matching original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"Chalons was\") and the same correction (\"Chalons, was\"), inserting a comma after \"Chalons\". This captures the core intended change precisely. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees,",
"corrected_text": "poplar-trees, hyphen",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "sodden-ed with",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:19:20.308930",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:48:44.156283",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same core change\u2014adding a comma after \"Chalons\" in the phrase \"Chalons was\" to make it \"Chalons, was\". This captures the essential intended correction. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees hyphen",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "sodden'ed with",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:19:58.804011",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:07:34.002395",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same core punctuation insertion, changing \"Chalons was\" to \"Chalons, was\" (inserting a comma after \"Chalons\"). This captures the essential intended edit exactly. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees hyphen",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "sodden'd with",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:27:51.999728",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:05:36.827813",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, inserting a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The original and corrected text match exactly, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly.\n\nText Content Accuracy: Both edits replace the same original text \"I hope so\" with the same corrected text \"I hoped so\". The core change (shifting verb tense from present to past) is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (as",
"corrected_text": "indeed thanked him from her heart (as",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness), she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length, said",
"corrected_text": "length, said",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:20:32.317312",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:49:22.481538",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"heart so\" to \"heart, so\", inserting a comma after \"heart\". The core intended punctuation change is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change from present to past tense. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (as",
"corrected_text": "indeed thanked him from her heart (as",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness), she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length, said",
"corrected_text": "length, said",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:21:03.593164",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:08:19.549963",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"heart so\") and the same corrected text (\"heart, so\"), inserting a comma after \"heart\". The core change is identical, with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"). The core intended change\u2014shifting from present tense \"hope\" to past tense \"hoped\"\u2014is captured identically. Therefore, the prediction matches the ground truth edit intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (as",
"corrected_text": "indeed thanked him from her heart (as",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness), she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length, said",
"corrected_text": "length, said",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:28:18.994398",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:06:18.285234",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 16,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers\". Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 18,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change of inserting a comma after \"hers\". Line numbers are ignored, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they",
"corrected_text": "\"They",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:21:39.994749",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:49:59.457083",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"hers and\" to \"hers, and\", inserting a comma after \"hers\". This captures the core intended punctuation correction with identical original and corrected text segments. Line numbers are irrelevant per instructions."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change (inserting a comma after \"hers\"). Line number differences are to be ignored, so the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: Both specify the same core change: inserting a comma after \"moment\" in the phrase \"moment with\", resulting in \"moment, with\". The original_text and corrected_text match the ground truth\u2019s intended punctuation change.\n\nLine numbers are ignored as instructed, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"beautiful\" in the phrase \"beautiful that\" to make it \"beautiful, that\". The original and corrected text strings match the ground truth\u2019s intended punctuation change.\nLine numbers are ignored, so the difference in line_number does not affect correctness."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "\"they",
"corrected_text": "\"They",
"observed_line_number": 35,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is capitalizing \u201cthey\u201d to \u201cThey.\u201d The prediction applies the same capitalization change but includes an additional leading quotation mark in both the original and corrected text (\"they \u2192 \"They). This extra context does not alter the core intended correction (lowercase t \u2192 uppercase T on \u201cthey\u201d). Therefore, the prediction captures the essential edit accurately."
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:22:09.846487",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:09:01.299329",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 16,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change of inserting a comma after \"hers\". Ignoring line numbers, the predicted edit matches the ground truth intention exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 18,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the core intended change of inserting a comma after \"hers\". Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"they",
"corrected_text": "\"They",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:28:53.863177",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:06:56.189924",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same core change, inserting a comma after \"shore\" so that \"shore and\" becomes \"shore, and\". Ignoring line numbers as instructed, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: Both edits replace \"that\" with \"than\". This captures the same core correction identified in the ground truth. Line number differences are ignored per instructions.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "you!\"",
"corrected_text": "you,\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "is now.\"",
"corrected_text": "is now.\" \"Thank you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:22:44.883255",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:50:36.692642",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Line numbers are ignored as instructed, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as a \"replacement\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: replacing \"that\" with \"than\". No discrepancy in original or corrected text content. Line number difference is ignored per instructions."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you!\"",
"corrected_text": "you,\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "is now.\"",
"corrected_text": "is now.\" \"Thank you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:23:23.402985",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:09:45.296098",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "the shore and",
"corrected_text": "the shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"shore\" in the phrase \"shore and\" \u2192 \"shore, and\". The prediction does the same core change, adding a comma after \"shore\" in a slightly longer context: \"the shore and\" \u2192 \"the shore, and\". This additional surrounding word (\"the\") does not alter the essential correction. Therefore, the predicted edit correctly captures the intended punctuation change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"that\" to \"than\". The prediction changes \"that once\" to \"than once\", which includes extra surrounding context but preserves the same core correction (\"that\" \u2192 \"than\") in the intended location. Therefore, the essential change is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!\"",
"corrected_text": "Thank you,\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the moonlight, the",
"corrected_text": "the moonlight the",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "is now.\"",
"corrected_text": "is now.\" \"Thank you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:29:38.337748",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:07:38.489313",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted change adds a comma after \"existence\" as well, shown with added surrounding context (\"existence at\" \u2192 \"existence, at\"). This captures the same core punctuation correction intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core punctuation change, inserting a comma immediately after \"motionless\" in the broader context (\"motionless when\" \u2192 \"motionless, when\"). Including the following word \"when\" does not alter the essential edit; it correctly captures the intention of adding a comma after \"motionless\"."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 18,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change within a larger context span (\"our view by\" \u2192 \"our view, by\"), which still places the comma immediately after \"view\". This captures the essential intended punctuation edit, with only additional surrounding words included as context."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with them;",
"corrected_text": "with them,",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it, to",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "c(cap) ity",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "each recurring with its accompanying monotony, always the same re-\nluctant return of",
"corrected_text": "to each recurring with its accompanying monotony, always the same re-\nluctant return of",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:24:04.826944",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:51:16.858783",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change, but includes additional surrounding context (\"existence at\" \u2192 \"existence, at\"). This still correctly captures the essential intended edit: inserting a comma after \"existence\"."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction expands the span to include the following word but performs the same core change: it inserts a comma immediately after \"motionless\" (\"motionless when\" \u2192 \"motionless, when\"). This captures the essential intended punctuation correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma after \"view\" within a longer phrase (\"our view by\" \u2192 \"our view, by\"). This includes extra surrounding context but captures the same core punctuation change\u2014inserting a comma immediately after \"view\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with them;",
"corrected_text": "with them,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still;",
"corrected_text": "stand-still,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence;",
"corrected_text": "existence,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "c(cap)ity",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "each recurring with its accompanying monotony, always the same re-\nluctant return of",
"corrected_text": "to each recurring with its accompanying monotony, always the same re-\nluctant return of",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:24:45.663500",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:10:34.286172",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction adds a comma immediately after \"existence\" as well, using a slightly larger text span (\"existence at\" \u2192 \"existence, at\"). This captures the same core punctuation change intended by the ground truth. Extra surrounding context in the prediction does not alter the essential edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change\u2014adding a comma immediately after \"motionless\"\u2014but includes additional surrounding context (\"motionless when\" \u2192 \"motionless, when\"). This captures the essential intended punctuation correction.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 18,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit inserts a comma after \"view\" as well, within a longer phrase (\"our view by\" \u2192 \"our view, by\"). This includes extra surrounding context but preserves the essential correction: adding a comma immediately after \"view.\" \u0644\u0630\u0644\u0643\u060c the prediction captures the core intended edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with them;",
"corrected_text": "with them,",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it, to",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "c(cap) ity",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "each recurring with its accompanying monotony, always the same re-\nluctant return of",
"corrected_text": "to each recurring with its accompanying monotony, always the same re-\nluctant return of",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:30:12.973160",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:08:19.961710",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as a \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and also inserts \"of\" to make the phrase \"driest of snorts\" (within the larger context \"of the ... .\"). The predicted edit likewise replaces \"dryest snorts\" with \"driest of snorts\", capturing the same core correction (spelling correction + insertion of \"of\" before \"snorts\"). Although the prediction omits surrounding context (\"of the\" and the final period), it still correctly represents the essential intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 31,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"italicize,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction target the same text (\"him\") and apply the same change (italicization) without altering the wording. Since line numbers are to be ignored, the core intended edit is captured correctly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: Both edits replace the same original text \"authorize\" with the same corrected text \"authorise\". This captures the identical core change (American to British spelling). Any difference in line number is to be ignored per instructions.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that, boiling-over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty, sticking",
"corrected_text": "at twenty; sticking",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "of him, than",
"corrected_text": "of him than",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:25:19.050028",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:51:54.315908",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts an additional \"of\" to make the phrase \"driest of snorts.\" The predicted edit likewise replaces \"dryest snorts\" with \"driest of snorts,\" capturing both the spelling correction (dryest \u2192 driest) and the insertion of \"of\". Although the prediction omits surrounding context (\"of the\" at the start and the final period), it preserves the core intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 31,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both indicate italicizing the same word \"him\" with no change in the text itself (original_text and corrected_text are both \"him\"), matching the intended formatting change. Line number differences are to be ignored, so the core edit is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"authorize\" with \"authorise\". This captures the same core intended change (US to UK spelling). Ignoring line numbers as instructed, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that, boiling-over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty, sticking",
"corrected_text": "at twenty; sticking",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "of him, than",
"corrected_text": "of him than",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:25:52.043544",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:11:21.710519",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as a \"replacement,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest snorts\" to \"driest of snorts\" (i.e., spelling correction dryest\u2192driest plus inserting \"of\"). The prediction makes the same core change: \"dryest snorts\" \u2192 \"driest of snorts.\" Although the ground truth includes surrounding context (\"of the\" and a period) and the prediction omits that context, the essential intended correction is captured accurately."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 31,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both identify the same target text \"him\" with no change to the characters, only the formatting (italicization). This captures the core intended edit.\n\nLine numbers are ignored per instructions, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 39,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"authorize\" with \"authorise\", which captures the same core correction (American to British spelling). Ignoring line numbers as instructed, the essential text change is identical."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that, boiling-over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty, sticking",
"corrected_text": "at twenty; sticking",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "of him, than",
"corrected_text": "of him, than",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T10:30:43.124424",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:08:58.753628",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The prediction likewise inserts a comma after \"For\" in the shorter span \"For Mr.\" \u2192 \"For, Mr.\". Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the same core punctuation change: adding a comma after \"For\" before \"Mr.\". Therefore, the intended edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"quickly that\") and the same correction (\"quickly, that\"), capturing the intended insertion of a comma after \"quickly.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me, not",
"corrected_text": "me; not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:26:19.394988",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T10:52:32.455954",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction also inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the core punctuation change at the correct location. Therefore, the essential edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"quickly\" (\"quickly that\" \u2192 \"quickly, that\"). The core punctuation correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me, not",
"corrected_text": "me; not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.6",
"date": "2026-03-08T11:26:43.285854",
"annotator": "Claude Opus 4.6",
"annotation_date": "2026-03-08T11:12:03.397334",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks.\" The prediction likewise inserts a comma after \"For\" in the corresponding phrase segment \"For Mr.\" \u2192 \"For, Mr.\" Although it includes less surrounding context (it omits \"Pancks\"), it correctly captures the core intended punctuation change (adding the comma after \"For\" before \"Mr.\")."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation: \"quickly that\" \u2192 \"quickly, that\", inserting a comma after \"quickly\". The core change is identical, with no mismatch in original or corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me; not",
"corrected_text": "me, not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2026-02-05",
"release_source": "https://www.anthropic.com/news/claude-opus-4-6",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Opus 4.7",
"model_id": "claude-opus-4.7",
"date": "2026-04-26T00:12:21.117662",
"shots": 2,
"config": {
"model_id": "claude-opus-4.7",
"display_name": "Claude Opus 4.7",
"shots": 2,
"temperature": 0.0,
"date": "2026-04-25",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:03.913445",
"file_results": [
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:19.108133",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:03.913445",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same correction (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the identical core change of inserting a comma after \"yonder\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:25.365692",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:02.981440",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a period to the end of \"Fellow Travellers\" to make it \"Fellow Travellers.\" The original and corrected texts are identical between ground truth and prediction, so the intended correction is captured precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the intended insertion of a comma after \"howling\". Therefore the core change matches perfectly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the identical core change of inserting a comma after \"yonder.\" Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:20.847541",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:31:52.670255",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: adding a period after \"Fellow Travellers\" (\"Fellow Travellers\" \u2192 \"Fellow Travellers.\"). The core intended correction is captured exactly with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both ground truth and prediction change \"howling over\" to \"howling, over\", inserting a comma after \"howling\". The core punctuation change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the core intended change of inserting a comma after \"yonder\". No discrepancy in the essential edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:22.590232",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:04.451332",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 8,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"No sir\" to \"No, sir\", inserting a comma after \"No\". This captures the core intended correction with identical original and corrected text.\nLine numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 27,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to capitalize the initial word in the phrase, changing \"why did he dine\" to \"Why did he dine\". The prediction captures this core change by changing \"why\" to \"Why\". Although it includes less surrounding context than the ground truth, it correctly represents the essential capitalization edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": 28,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"cold stones!\" and the same corrected text \"cold stones?\"\u2014changing an exclamation mark to a question mark. This captures the core intended edit with no deviation. Line number differences are ignored as instructed."
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:22.964073",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:01.289009",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 8,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"No sir\" to \"No, sir\" (inserting a comma after \"No\"). The core correction is identical. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is capitalizing the initial word in the phrase: \"why did he dine\" \u2192 \"Why did he dine.\" The predicted edit focuses on the essential change within that phrase: \"why\" \u2192 \"Why.\" Although it includes less surrounding context, it captures the core intended correction (capitalizing \"why\"). Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change from \"cold stones!\" to \"cold stones?\" with identical original and corrected text. Ignoring line numbers, the intended edit is captured exactly."
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:21.667966",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:31:50.504740",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both apply the same core change, inserting a comma after \"No\" in the phrase \"No sir\" to make it \"No, sir\". The original_text and corrected_text match the ground truth\u2019s intended punctuation change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase (\"why did he dine\" \u2192 \"Why did he dine\"). The prediction captures the same core change by converting \"why\" \u2192 \"Why\". Although it includes less surrounding context than the ground truth, it correctly identifies and applies the essential capitalization change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: replacing the exclamation mark with a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". There is no discrepancy in original or corrected text. Therefore, the predicted edit correctly captures the intended correction."
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:21.102473",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:06.512498",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"capitalization,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"if he\" to \"If he,\" capturing the identical core correction (capitalizing the initial 'i'). No additional or missing changes are present."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction label the change as a \"replacement,\" so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes the word \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam.\" The predicted edit captures the same core substitution (\"said\" \u2192 \"asked\") even though it includes less surrounding context (it omits \"Clennam\" from the specified span). Since the essential intended change is the replacement of \"said\" with \"asked,\" the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:15.551415",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:13:58.072142",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:20.303857",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:31:45.930621",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 33,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction specify the same core change: \"if he\" \u2192 \"If he\" (capitalizing the initial 'i' to 'I'). This captures the intended correction precisely. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\" edits, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes the verb from \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the same core substitution (\"said\" \u2192 \"asked\"), even though it omits the surrounding context (\"Clennam\"). Since the essential intended change is the verb replacement, the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:09:25.847428",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:04.067513",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"high road\" is corrected to \"high-road\" by inserting a hyphen. The original_text and corrected_text are identical between ground truth and prediction, capturing the intended edit precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit type matches exactly: both are labeled as \"punctuation\". Text content also matches the ground truth\u2019s core change: inserting a comma after \"materials\" so that \"materials costing\" becomes \"materials, costing\". No divergence in the original or corrected text, so the prediction fully captures the intended edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same correction (\"about, and\"), capturing the identical core change\u2014adding a comma after \"about\". Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 20,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change: inserting a comma after \"earnestly\". Line number differences are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 26,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same change from \"said when\" to \"said, when,\" inserting a comma after \"said.\" This captures the core intended correction exactly. Line number differences are to be ignored."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion did",
"corrected_text": "tion, did",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:29.745000",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:00.459185",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: replacing \"high road\" with the hyphenated form \"high-road\". The original and corrected texts are identical between ground truth and prediction, capturing the intended punctuation change precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original span (\"materials costing\") and the same correction (\"materials, costing\"), inserting a comma after \"materials.\" This captures the core intended change with no discrepancies."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same corrected text (\"about, and\"), capturing the identical core change\u2014adding a comma after \"about.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 19,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"earnestly telling\" \u2192 \"earnestly, telling\", i.e., inserting a comma after \"earnestly\". This captures the core intended change with no discrepancy in the edited text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"said when\" and the same corrected text \"said, when\", capturing the identical core change (inserting a comma after \"said\"). Line number differences are to be ignored, so the prediction matches the ground truth\u2019s intended correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both edits change \"said\" to \"said,\" by adding a comma immediately after the word. This captures the same core correction. Line numbers are ignored as instructed, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 32,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same corrected text (\"Clennam, in\"), capturing the core intended change of inserting a comma after \"Clennam.\" Line numbers are ignored as instructed."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association",
"corrected_text": "association,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:21.219569",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:31:52.346737",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change from \"high road\" to \"high-road\" (adding a hyphen). The original and corrected texts are identical between ground truth and prediction, so the intended edit is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both identify the same original phrase (\"materials costing\") and the same correction (\"materials, costing\"), inserting a comma after \"materials\". The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma to change \"about and\" to \"about, and\". The original and corrected text match the ground truth\u2019s intention with no deviation. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 28,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction also adds a comma immediately after \"said\" (\"said when\" \u2192 \"said, when\"). Although the prediction includes extra surrounding context (the following word \"when\"), it captures the same core punctuation change intended in the ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion did",
"corrected_text": "tion, did",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:21.110192",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:00.341421",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:26.951963",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:13:58.266240",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:26.067309",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:31:50.010966",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is to add a period at the end of \"Sun and Shadow\" (\"Sun and Shadow\" \u2192 \"Sun and Shadow.\"). The prediction also results in the same corrected text ending with a period. Although the prediction\u2019s original_text includes a comma (\"Sun and Shadow,\") implying a comma-to-period replacement rather than period insertion, it still captures the core intended correction: the phrase should end with a period. Therefore, the essential edit matches the ground truth\u2019s intention."
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:40.143287",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:04.710667",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The original and corrected texts are identical between ground truth and prediction, capturing the core intended change precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"dull autumn\") and the same corrected text (\"dull, autumn\"), capturing the identical core change: inserting a comma after \"dull.\" Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"man slowly\") and the same correction (\"man, slowly\"), capturing the identical core change\u2014adding a comma after \"man.\" Line numbers are to be ignored, so this difference does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2. Text Content Accuracy: Both specify the same core punctuation change: inserting a comma after \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). The original and corrected text strings are identical between ground truth and prediction. Line number differences are to be ignored, so the prediction captures the intended edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\" edits, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"sodden\" to \"soddened\". The prediction changes \"sodden with\" to \"soddened with\", which includes additional surrounding context (\"with\") but preserves the same core change from \"sodden\" to \"soddened\". This accurately captures the intended correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:30.796647",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:03.628194",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The original and corrected text are identical to the ground truth\u2019s intention. Therefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. Text Content Accuracy: Both specify the same transformation from \"dull autumn\" to \"dull, autumn\", inserting a comma after \"dull\". The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"poplar trees\") and the same corrected text (\"poplar-trees,\")\u2014adding a hyphen between \"poplar\" and \"trees\" and adding a trailing comma. The core change is identical to the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both edits apply the same core punctuation change, inserting a comma after \"man\" to convert \"man slowly\" into \"man, slowly\". The original and corrected text strings match the ground truth\u2019s intention exactly.\n\nLine numbers are ignored as instructed, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). This captures the essential correction intended in the ground truth. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:35.408877",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:31:54.091526",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text \"Let Loose\" and corrected text \"Let Loose.\", capturing the identical core change (adding a period). Therefore the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"dull autumn\" to \"dull, autumn\", inserting a comma after \"dull\". The core change is identical with no discrepancies. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same core punctuation change from \"man slowly\" to \"man, slowly\" (inserting a comma after \"man\"). This captures the essential intended correction. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same core change: inserting a comma after \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). The essential punctuation correction is captured identically. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"sodden\" to \"soddened\". The prediction changes \"sodden with\" to \"soddened with\", which includes extra surrounding context (\"with\") but preserves the same core change: \"sodden\" \u2192 \"soddened\". Therefore, the predicted edit captures the essential intended correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:21.770768",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:09.356734",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 11,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"heart so\") and the same corrected text (\"heart, so\"), capturing the identical core change\u2014adding a comma after \"heart.\" Ignoring line numbers, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did,",
"corrected_text": "(as she did indeed,",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:26.372059",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:02.321189",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"heart so\" to \"heart, so\", inserting a comma after \"heart\". The core correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart (as she did",
"corrected_text": "thanked him from her heart (as indeed she did",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:10:57.800867",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:31:50.813258",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 11,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"heart so\" to \"heart, so,\" inserting a comma after \"heart.\" This captures the same core correction with identical original and corrected text. Line numbers are ignored as instructed, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart (as she did",
"corrected_text": "thanked him from her heart indeed (as she did",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:11:26.150180",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:08.413618",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth. Text Content Accuracy: Both ground truth and prediction change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers.\" The core punctuation correction is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"moment with\" \u2192 \"moment, with\", inserting a comma after \"moment\". This captures the core intended correction with identical original and corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction change \"beautiful that\" to \"beautiful, that\" by inserting a comma after \"beautiful.\" This captures the core intended correction with identical original and corrected text. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "directly, \"they",
"corrected_text": "directly, \"They",
"observed_line_number": 35,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change \"they\" to \"They\" (capitalizing the initial letter). The prediction changes \"directly, \\\"they\" to \"directly, \\\"They,\" which includes extra surrounding context but performs the same core capitalization change on the word \"they.\" Since the essential correction (they \u2192 They) is captured, the text content matches the ground truth\u2019s intention."
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:11:15.359528",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:01.779945",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both edits make the same core change, inserting a comma after \"hers\" to change \"hers and\" into \"hers, and.\" This captures the intended correction precisely. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same correction (\"hers, matters\"), capturing the identical core change\u2014adding a comma after \"hers\". Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"moment with\" to \"moment, with,\" inserting a comma after \"moment.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"beautiful\" in the phrase \"beautiful that,\" i.e., \"beautiful that\" \u2192 \"beautiful, that.\" This captures the essential intended correction with no discrepancy in the text change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 34,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: The predicted type is \"capitalization,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change the same word from \"they\" to \"They,\" capturing the identical core correction (capitalizing the initial letter). Line number differences are ignored as instructed.\nTherefore, the predicted edit matches the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:11:31.834993",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:31:47.462752",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original segment (\"hers and\") and the same correction (\"hers, and\"), inserting a comma after \"hers\". This captures the core intended change precisely. Line numbers are irrelevant per instructions."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change\u2014adding a comma after \"hers.\" Line numbers are to be ignored, so the mismatch there does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original_text (\"moment with\") and the same corrected_text (\"moment, with\"), capturing the core intended change\u2014adding a comma after \"moment.\" Ignoring line numbers, the prediction matches the ground truth edit precisely."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same corrected text (\"beautiful, that\"), capturing the identical core change: inserting a comma after \"beautiful\". Ignoring line numbers as instructed, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 35,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same core change: original_text \"they\" corrected to \"They\". This captures the essential intended edit with identical before/after text.\n\nLine numbers are ignored per instructions, so the mismatch there does not affect correctness."
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:11:26.539892",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:42.120551",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original segment (\"shore and\") and the same correction (\"shore, and\"), capturing the identical core punctuation insertion (a comma after \"shore\"). Line numbers are ignored as instructed, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "greater things that once were",
"corrected_text": "greater things than once were",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes the word \"that\" to \"than\". The predicted edit changes the phrase \"greater things that once were\" to \"greater things than once were\", which includes the same core substitution (\"that\" \u2192 \"than\") with additional surrounding context. This correctly captures the intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in our fancy",
"corrected_text": "that, in our fancy,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:11:30.334417",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:36.782400",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"shore and\" to \"shore, and\" by inserting a comma after \"shore\". The original and corrected text strings match the ground truth\u2019s core change exactly. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "things that once",
"corrected_text": "things than once",
"observed_line_number": 35,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"that\" to \"than\". The predicted edit changes the phrase \"things that once\" to \"things than once\", which contains the same core correction (replacing \"that\" with \"than\") while including extra surrounding context. This captures the essential intended change.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in our fancy",
"corrected_text": "that, in our fancy,",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:11:39.508236",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:32:32.676558",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"shore\" in the phrase \"shore and\" to become \"shore, and.\" The original_text and corrected_text match the ground truth\u2019s intended modification. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in our fancy",
"corrected_text": "that, in our fancy,",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "things that once were",
"corrected_text": "things than once were",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:11:43.841517",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:47.428462",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"city\" is corrected to \"City\". There is no deviation in original or corrected text. Therefore, the predicted edit captures the ground truth intention precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit applies the same core change, adding a comma immediately after \"existence\" within a slightly longer span (\"existence at\" \u2192 \"existence, at\"). This includes extra context but preserves the essential intended punctuation correction.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction adds the same comma after \"motionless\" but includes the following word for context (\"motionless when\" \u2192 \"motionless, when\"). This captures the same core punctuation change intended by the ground truth.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change within a larger span (\"view by\" \u2192 \"view, by\"), which still places the comma immediately after \"view\". The additional context (including \"by\") does not alter the essential intended punctuation change.\n\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it, to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "return of",
"corrected_text": "return to of",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liegelord",
"corrected_text": "liege lord",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:12:21.117662",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:38.963281",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"city\" to \"City\", capturing the identical core change (capitalizing the first letter). No discrepancies in original or corrected text are present."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates inserting a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction expands the span to include the following word, changing \"existence at\" \u2192 \"existence, at\", which still represents the same core change: adding a comma immediately after \"existence\". The extra context word does not alter the intended punctuation correction.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction makes the same essential punctuation change, just with additional surrounding context (\"motionless when\" \u2192 \"motionless, when\"). This correctly captures the core intended edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma after \"view\" as well, shown with additional surrounding context (\"view by\" \u2192 \"view, by\"). This captures the same essential punctuation insertion at the same location relative to the word \"view.\""
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to replace the punctuation immediately following \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction applies the same core punctuation change by making \"existence\" followed by a semicolon (\"existence is\" \u2192 \"existence; is\"), adding surrounding context (the following word \"is\"). Even though the predicted original snippet omits the comma explicitly, the essential intended correction\u2014placing a semicolon after \"existence\"\u2014is correctly captured."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "of the",
"corrected_text": "of to the",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liegelord",
"corrected_text": "liege lord",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:12:01.822878",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:32:36.813477",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core transformation: changing \"city\" to \"City\". There is no discrepancy in original or corrected text. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates inserting a comma immediately after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change, adding a comma after \"existence\" while including extra surrounding context (\"existence at\" \u2192 \"existence, at\"). This preserves the essential intended correction.\n\nTherefore, the predicted edit correctly captures the ground truth edit\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction expands the span to include the following word (\"motionless when\" \u2192 \"motionless, when\"), but it still performs the same core punctuation insertion: a comma immediately after \"motionless.\" This correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and predicted edit are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction adds a comma after \"view\" as well, but includes extra surrounding context (\"view by\" \u2192 \"view, by\"). This still captures the same core punctuation insertion intended by the ground truth.\n\nTherefore, the predicted edit correctly reflects the ground truth\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it, to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "of the",
"corrected_text": "of to the",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liegelord",
"corrected_text": "liege lord",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:12:01.035857",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:43.704342",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly.\nText Content Accuracy: Both specify the same target text \"him\" with no change in wording (only formatting to italics). The predicted edit captures the same core formatting change intended by the ground truth.\nTherefore, the prediction correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"now\" in the phrase \"now I\" to make it \"now, I.\" There is no discrepancy in original or corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\nText Content Accuracy: Both replace the exact same original text (\"authorize\") with the same corrected text (\"authorise\"). The core change (American to British spelling) is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:12:08.419577",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:40.819855",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change: \"twenty;\" is corrected to \"twenty:\". Since the essential edit is identical, the prediction matches the ground truth's intention. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same target text, \"him\" \u2192 \"him\", indicating the intention is to italicize the word without changing its spelling. The predicted edit captures the same core change as the ground truth.\n\nTherefore, the predicted edit correctly matches the ground truth in both type and content."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have",
"corrected_text": "now, I have",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The core intended change in the ground truth is inserting a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The prediction performs the same essential punctuation change, simply including extra surrounding context (\"now I have\" \u2192 \"now, I have\"). This preserves the exact intended correction (comma insertion after \"now\")."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same replacement of \"authorize\" with \"authorise\", capturing the identical core change (US to UK spelling). Ignoring line numbers as instructed, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "the dryest snorts",
"corrected_text": "the driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:12:12.885961",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:32:41.395573",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction apply italicization to the same token, with original_text \"him\" and corrected_text \"him\" (indicating a formatting change only). The core intended change\u2014italicizing \"him\"\u2014is captured identically.\n\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": 35,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core correction, changing \"now I\" to \"now, I\" by inserting a comma after \"now.\" This fully captures the ground truth intention. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 39,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2. Text Content Accuracy: Both edits replace the original text \"authorize\" with the corrected text \"authorise\". This captures the core intended change (American to British spelling). Line number differences are to be ignored, so the content matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:12:05.322933",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:09:44.655922",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"For Mr. Pancks\" to \"For, Mr. Pancks\" by inserting a comma after \"For\". The core punctuation change is identical in the prediction and the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"quickly\" in the phrase \"quickly that\" to make \"quickly, that\". This captures the core intended correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "insertion",
"original_text": "making me, not",
"corrected_text": "making me a, not",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches.\n\n2. Text Content Accuracy: The ground truth inserts \"a\" after \"me\" (\"me,\" \u2192 \"me a,\"). The predicted edit also inserts \"a\" after \"me\" within a longer phrase (\"making me, not\" \u2192 \"making me a, not\"). Although the prediction includes additional surrounding context, it captures the same core change: inserting \"a\" between \"me\" and the comma. Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:12:03.409124",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:14:44.337185",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The core intended change in the ground truth is inserting a comma after the word \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction likewise inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it omits \"Pancks\"), it still captures the essential punctuation change at the correct location. Therefore, it matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"quickly\" in the phrase \"quickly that\" \u2192 \"quickly, that\". The original_text and corrected_text match exactly, so the essential edit is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "making me not",
"corrected_text": "making me a not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.7",
"date": "2026-04-26T00:12:11.552778",
"annotator": "Claude Opus 4.7",
"annotation_date": "2026-04-25T23:32:36.667012",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"For Mr. Pancks\" to \"For, Mr. Pancks\", inserting a comma after \"For\". The core punctuation change is identical, with no discrepancies in original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 13,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both edits change \"quickly that\" to \"quickly, that,\" inserting a comma after \"quickly.\" This captures the same core punctuation correction intended by the ground truth. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "making me not",
"corrected_text": "making me a not",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2026-04-16",
"release_source": "https://www.anthropic.com/news/claude-opus-4-7",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Opus 4.8",
"model_id": "claude-opus-4.8",
"date": "2026-07-23T15:11:43.568625",
"shots": 2,
"config": {
"model_id": "claude-opus-4.8",
"display_name": "Claude Opus 4.8",
"shots": 2,
"temperature": 0.0,
"date": "2026-07-23",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:17.759132",
"file_results": [
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:04.397109",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:17.759132",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"howling over\" to \"howling, over\" by inserting a comma after \"howling.\" The core punctuation change is identical, with matching original and corrected text.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"yonder to-day\" to \"yonder, to-day\" by inserting a comma after \"yonder.\" The original and corrected texts match exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:09:52.282814",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:17.994662",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original phrase \"howling over\" and the same correction \"howling, over\", capturing the identical core change (inserting a comma after \"howling\"). Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"yonder to-day\" and the same correction \"yonder, to-day\", inserting a comma after \"yonder\". The core punctuation change is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:09:55.177089",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:14.356977",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the intended insertion of a comma after \"yonder\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:09:53.607331",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:24.522500",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir,",
"corrected_text": "No, sir,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:09:53.780354",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:17.154894",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir,",
"corrected_text": "No, sir,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:09:52.383594",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:19.106570",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "\"No sir,",
"corrected_text": "\"No, sir,",
"observed_line_number": 11,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth correction inserts a comma after \"No\" (\"No sir\" \u2192 \"No, sir\"). The prediction likewise changes \"No sir\" to \"No, sir\" within a larger quoted/contextual span (\"\\\"No sir,\" \u2192 \"\\\"No, sir,\"). Although the prediction includes extra surrounding punctuation (opening quote and an existing trailing comma), it captures the essential intended change: adding the comma after \"No\".\n\nTherefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"why did",
"corrected_text": "\"Why did",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!\"",
"corrected_text": "cold stones?\"",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:09:44.029529",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:12.690979",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 48,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 51,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:09:50.732278",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:20.980294",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 45,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 49,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:09:44.024951",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:07.059404",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 45,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 48,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:11.498767",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:17.608116",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same core change from \"high road\" to \"high-road\" (adding a hyphen between the words). There is no discrepancy in original or corrected text. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"materials\" in the phrase \"materials costing,\" yielding \"materials, costing.\" The original_text and corrected_text match the ground truth\u2019s intended punctuation change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tossed about and",
"corrected_text": "tossed about, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "association did",
"corrected_text": "association did",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 42,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:10.544903",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:23.173706",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original text \"high road\" and the same corrected text \"high-road,\" capturing the identical core change (hyphenation). No discrepancy in the essential edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"materials\" in the phrase \"materials costing\" to make it \"materials, costing.\" The original_text and corrected_text match the ground truth\u2019s intended punctuation change. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:27.164499",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:12.202920",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change from \"high road\" to \"high-road\" (adding a hyphen). There is no deviation in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change\u2014adding a comma after \"materials\". Line number differences are to be ignored, so the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said;",
"corrected_text": "said,",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:40.606780",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:12.446049",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to make the phrase end with a period (\"Sun and Shadow\" \u2192 \"Sun and Shadow.\"). The prediction also results in the same corrected text ending with a period (\"Sun and Shadow,\" \u2192 \"Sun and Shadow.\"). Although the predicted original text includes a comma that is not present in the ground truth original text, the core intended correction\u2014ensuring a period at the end of the phrase\u2014is captured correctly."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter 1,",
"corrected_text": "Chapter 1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:22.977511",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:18.057631",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter 1",
"corrected_text": "Chapter 1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:29.639363",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:08.057210",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to end the phrase \"Sun and Shadow\" with a period (i.e., add/ensure a final period). The prediction also results in \"Sun and Shadow.\" as the corrected text. Although the prediction\u2019s original_text includes a comma (\"Sun and Shadow,\") whereas the ground truth original_text has no punctuation (\"Sun and Shadow\"), the core intended correction\u2014making the ending punctuation a period\u2014is accurately captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I,",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:39.152743",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:25.662427",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a period to \"Let Loose\" to make it \"Let Loose.\". The original and corrected texts are identical between ground truth and prediction, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, inserting a comma after \"dull\" in the phrase \"dull autumn\" \u2192 \"dull, autumn\". The original and corrected texts are identical between ground truth and prediction, so the intended edit is captured correctly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 7,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"poplar trees\") and the same corrected text (\"poplar-trees,\") indicating the identical core change: adding a hyphen and a comma. Line numbers are ignored as instructed, so this does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth inserts a comma after \"Chalons\" in the phrase \"Chalons was\" \u2192 \"Chalons, was.\" The prediction also captures the same core change by editing \"Chalons\" \u2192 \"Chalons,\" (adding a comma immediately after \"Chalons\"). Although the prediction omits the following word \"was\" from its context, it still correctly represents the essential punctuation insertion intended by the ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream",
"corrected_text": "stream,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man",
"corrected_text": "One man,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:54.397339",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:22.194965",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"Let Loose\" to \"Let Loose.\" (adding a period). The core intended correction is captured exactly with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both ground truth and prediction change \"dull autumn\" to \"dull, autumn\" by inserting a comma after \"dull\". The core intended punctuation change is captured exactly with the same original and corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"poplar trees\") and the same corrected text (\"poplar-trees,\") capturing the intended hyphen insertion and comma addition. The added notes in the prediction do not change the edit content. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"man\" (turning \"man slowly\" into \"man, slowly\"). The predicted edit \"man\" \u2192 \"man,\" captures the same core punctuation insertion at the same location (immediately after \"man\"), even though it omits the following word for context. This matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change inserts a comma after the word \"Chalons\" in the phrase \"Chalons was\" \u2192 \"Chalons, was\". The predicted edit \"Chalons\" \u2192 \"Chalons,\" captures the same core punctuation change (adding a comma immediately after \"Chalons\"), just with less surrounding context. This matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream",
"corrected_text": "stream,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:09.127723",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:13.525640",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The core change is identical, with no discrepancies in original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"dull autumn\") and the same corrected text (\"dull, autumn\"), capturing the identical core change: inserting a comma after \"dull\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "row of poplar trees",
"corrected_text": "row of poplar-trees,",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"poplar trees\" to \"poplar-trees,\" (adds a hyphen between \"poplar\" and \"trees\" and adds a trailing comma). The prediction applies the same core change within a slightly larger context: \"row of poplar trees\" \u2192 \"row of poplar-trees,\". Including extra surrounding words is acceptable, and the essential modification (hyphenation and comma) matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The core intended change is inserting a comma after the word \"man\" (\"man slowly\" \u2192 \"man, slowly\"). The prediction applies the same punctuation change within a slightly larger span of text (\"One man slowly\" \u2192 \"One man, slowly\"). Including extra surrounding context is acceptable, and the essential edit is correctly captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "towards Chalons was",
"corrected_text": "towards Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The core intended change in the ground truth is inserting a comma after \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). The predicted edit makes the same core change, just with additional surrounding context (\"towards Chalons was\" \u2192 \"towards Chalons, was\"). This still captures the essential punctuation insertion at the correct location.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream, like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:35.661749",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:15.544510",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"heart so\") and the same corrected text (\"heart, so\"), inserting a comma after \"heart.\" This captures the core intended change precisely. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart (as she did,",
"corrected_text": "thanked him from her heart (as she did indeed,",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:36.588283",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:18.212950",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation: inserting a comma after \"heart\" so that \"heart so\" becomes \"heart, so\". This captures the core intended change with identical original and corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart (as she did",
"corrected_text": "thanked him from her heart indeed (as she did",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so; I am not disappointed",
"corrected_text": "I hoped so; I am not disappointed",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:00.205070",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:14.191247",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"heart so\" to \"heart, so\", inserting a comma after \"heart\". This captures the same core punctuation correction as the ground truth. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "I hope so; I am not disappointed.",
"corrected_text": "I hope so; I am not disappointed. I hoped so",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart (as she did",
"corrected_text": "thanked him from her heart indeed (as she did",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:51.970566",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:50.941378",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "from hers and",
"corrected_text": "from hers, and",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters",
"corrected_text": "on hers, matters",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "a moment with",
"corrected_text": "a moment, with",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so beautiful that",
"corrected_text": "so beautiful, that",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 45,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:10:55.260273",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:52.523390",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:00.634591",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:49.958127",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:15.672499",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:11:51.182931",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation from an exclamation point to a comma in the phrase \"Thank you\" (\"Thank you!\" \u2192 \"Thank you,\"). The prediction captures the same core change, replacing ! with , in the same phrase, but includes surrounding quotation marks as additional context (\"Thank you!\" \u2192 \"Thank you,\"). This added context does not alter the essential intended correction. Therefore, the text content matches the ground truth's intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "greater things that once",
"corrected_text": "greater things than once",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that, in our fancy, whatever",
"corrected_text": "that in our fancy whatever",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:08.907509",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:53.918470",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "greater things that once were",
"corrected_text": "greater things than once were",
"observed_line_number": 44,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that, in our fancy, whatever",
"corrected_text": "that in our fancy whatever",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:07.442178",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:51.012969",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that, in our fancy, whatever",
"corrected_text": "that in our fancy whatever",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the shore and",
"corrected_text": "the shore, and",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "greater things that once",
"corrected_text": "greater things than once",
"observed_line_number": 45,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:30.165049",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:12:06.618304",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify changing \"city\" to \"City\", which captures the identical core correction. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with them;",
"corrected_text": "with them,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence at",
"corrected_text": "busy existence, at",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it, to",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liege lord",
"corrected_text": "liege lord",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX.",
"corrected_text": "Chapter XXIX",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "DREAMING.",
"corrected_text": "DREAMING",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "reluctant return of",
"corrected_text": "reluctant return of",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:31.245206",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:58.345215",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\". The original and corrected texts match exactly, capturing the intended punctuation insertion."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction change \"DREAMING\" to \"DREAMING.\" by adding a period. This captures the core intended correction. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly. Text Content Accuracy: Both specify the same core change, converting \"city\" to \"City.\" Line numbers are to be ignored, so the differing line_number does not affect correctness. Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it, to",
"corrected_text": "it; to",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still, to",
"corrected_text": "stand-still; to",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence, is",
"corrected_text": "existence; is",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liegelord",
"corrected_text": "liege lord",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "reluctant return of",
"corrected_text": "reluctant return to of",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:32.627984",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:59.376480",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core correction: adding a period to change \"Chapter XXIX\" to \"Chapter XXIX.\". The original and corrected texts are identical between ground truth and prediction, capturing the intended punctuation insertion precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"DREAMING\" to \"DREAMING.\" by adding a period. This captures the same core punctuation correction. Line numbers are ignored as instructed, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The ground truth changes \"city\" to \"City.\" The predicted edit also changes \"city\" to \"City,\" capturing the same core capitalization correction with identical original and corrected text.\n\nLine numbers are ignored as instructed, so the difference in line_number does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "reluctant return of the",
"corrected_text": "reluctant return to the",
"observed_line_number": 6,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\" edits, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces the word \"of\" with \"to\". The predicted edit replaces the phrase \"reluctant return of the\" with \"reluctant return to the\", which captures the same core change (\"of\" \u2192 \"to\") while including additional surrounding context. This aligns with the ground truth\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence at",
"corrected_text": "busy existence, at",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it, to",
"corrected_text": "it; to",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still, to",
"corrected_text": "stand-still; to",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence, is",
"corrected_text": "existence; is",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liegelord",
"corrected_text": "liege lord",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:34.893856",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:12:04.410253",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and adds \"of\" to form \"driest of snorts\" (within the larger phrase \"of the ... .\"). The prediction captures the same core transformation: \"dryest snorts\" \u2192 \"driest of snorts.\" Although the prediction omits surrounding context (\"of the\" and the final period), it correctly represents the essential edit intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "that boiling-over",
"corrected_text": "that boiling-over",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty, sticking",
"corrected_text": "at twenty; sticking",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "out of him than",
"corrected_text": "out of him, than",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "\"Your own recompense",
"corrected_text": "\"Your own recompense",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have",
"corrected_text": "now, I have",
"observed_line_number": 45,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 49,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 49,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam, \"not",
"corrected_text": "said Clennam; \"not",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:39.084264",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:23:57.112004",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" \u2192 \"driest\" and inserts \"of\" to make the phrase \"driest of snorts\" (within the larger context \"of the ... .\"). The predicted edit captures the same core transformation: \"dryest snorts\" \u2192 \"driest of snorts.\" Although the prediction omits surrounding context (\"of the\" and the period), it still represents the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "that boiling-over",
"corrected_text": "that of boiling-over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty. sticking",
"corrected_text": "at twenty; sticking",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "out of him than",
"corrected_text": "out of him, than",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have",
"corrected_text": "now, I have",
"observed_line_number": 47,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 51,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "authorize you now",
"corrected_text": "authorise you, now",
"observed_line_number": 51,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam, \"not",
"corrected_text": "said Clennam; \"not",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:43.568625",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:58.269279",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are marked as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts \"of\" to make the phrase \"driest of snorts\" (within the larger context \"of the ... .\"). The predicted edit likewise changes \"dryest\" \u2192 \"driest\" and produces \"driest of snorts\". Although the prediction omits surrounding context (\"of the\" and the final period), it correctly captures the core intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "that boiling-over",
"corrected_text": "that boiling-over",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty sticking",
"corrected_text": "at twenty; sticking",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "bubbling out of him",
"corrected_text": "bubbling out of him",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have",
"corrected_text": "now, I have",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 50,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 50,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam \"not",
"corrected_text": "said Clennam; \"not",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:34.755282",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T11:12:05.278290",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change inserts a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The predicted edit likewise inserts a comma after \"For\" in a slightly shorter span, \"For Mr.\" \u2192 \"For, Mr.\". Although it omits the trailing word \"Pancks\", it captures the core intended punctuation change (adding the comma after \"For\" before \"Mr.\"). Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "making me, not",
"corrected_text": "making me, a not",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:34.887364",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T12:24:02.194892",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same correction (inserting a comma after \"For\" to make \"For, Mr. Pancks\"). The core punctuation change is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-opus-4.8",
"date": "2026-07-23T15:11:41.483870",
"annotator": "Claude Opus 4.8",
"annotation_date": "2026-07-23T13:23:55.671533",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"For\" in the phrase \"For Mr. Pancks\" to become \"For, Mr. Pancks\". The core punctuation correction is identical."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2026-05-28",
"release_notes": "Anthropic released Claude Opus 4.8 in late May 2026; the OpenRouter listing appeared 2026-05-27.",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit. Opus 4.8's score is depressed by unreliable line counting: it identifies most edits correctly (166 of ~200 match ground truth by text) but reports noisy line numbers that drift progressively down the page and vary between runs, so the judge rejects the matches."
},
{
"model_name": "Claude Opus 5 (OR)",
"model_id": "or_claude_opus_5",
"date": "2026-09-06T04:06:18.140146",
"shots": 2,
"config": {
"model_id": "or_claude_opus_5",
"display_name": "Claude Opus 5 (OR)",
"shots": 2,
"temperature": 0.0,
"reasoning_effort": "high",
"date": "2026-09-06",
"notes": "Benchmark run with 2-shot learning at high reasoning effort"
},
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:27.021401",
"file_results": [
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:36.618826",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:27.021401",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The core punctuation change is captured exactly with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original phrase (\"howling over\") and the same correction (\"howling, over\"), capturing the identical core change: insertion of a comma after \"howling\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"yonder to-day\" and the same correction \"yonder, to-day\", inserting a comma after \"yonder\". The core punctuation change is identical.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same transformation of the word \"Sir\" to \"sir\", capturing the identical core change (lowercasing the initial letter). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core change, inserting a comma after \"pockets\" (\"pockets and\" \u2192 \"pockets, and\"). The corrected_text matches exactly, capturing the intended punctuation insertion."
}
],
"file_id": "003"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:40.855524",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:31.248495",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth. Text Content Accuracy: Both ground truth and prediction change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The core change and the specific text involved match exactly. Therefore, the prediction correctly captures the intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"howling over\" to \"howling, over,\" inserting a comma after \"howling.\" This captures the identical core punctuation change with matching original and corrected text.\n\nTherefore, the prediction correctly reflects the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the identical core change\u2014adding a comma after \"yonder\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"Sir\" to \"sir\". The core intended correction (lowercasing the initial letter) is captured precisely with no discrepancy. Therefore, the predicted edit matches the ground truth intent."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction change \"pockets and\" to \"pockets, and\" by inserting a comma after \"pockets.\" This captures the core intended punctuation correction with identical original and corrected text."
}
],
"file_id": "003"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:44.884491",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:02:49.847907",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly. Text Content Accuracy: Both edits apply the same core change\u2014adding a period to convert \"Fellow Travellers\" to \"Fellow Travellers.\". Since both the type and the essential text modification match, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling",
"corrected_text": "howling,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is to insert a comma after the word \"howling\" in the phrase \"howling over\" (i.e., \"howling over\" \u2192 \"howling, over\"). The predicted edit also inserts a comma after \"howling\" (\"howling\" \u2192 \"howling,\"). Although the prediction includes less surrounding context (it omits \"over\"), it captures the core intended punctuation change: adding a comma immediately after \"howling.\""
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder",
"corrected_text": "yonder,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"yonder\" in the phrase \"yonder to-day\" (making it \"yonder, to-day\"). The predicted edit \"yonder\" \u2192 \"yonder,\" captures this same core punctuation change (adding a comma immediately after \"yonder\"), even though it omits the following word \"to-day\". Since the essential intended correction is the comma insertion after \"yonder\", the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: \"Sir\" is corrected to \"sir\" (lowercasing the initial letter). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets",
"corrected_text": "pockets,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"pockets\" in the phrase \"pockets and\" \u2192 \"pockets, and.\" The predicted edit \"pockets\" \u2192 \"pockets,\" captures the same core punctuation change (adding a comma immediately after \"pockets\"). Although it omits the following word \"and,\" it still reflects the essential intended correction."
}
],
"file_id": "003"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:35.971164",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:28.887570",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"No sir, I have got Maggy with me.\"",
"corrected_text": "\"No, sir, I have got Maggy with me.\"",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth correction is inserting a comma between \"No\" and \"sir\" (\"No sir\" \u2192 \"No, sir\"). The predicted edit applies the same core change within a longer quoted sentence (\"No sir, I have got...\" \u2192 \"No, sir, I have got...\"). Although it includes additional surrounding context, the essential punctuation change (adding the comma after \"No\") matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\"why did he dine to-",
"corrected_text": "\"Why did he dine to-",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the initial \"why\" to \"Why\" in the phrase \"why did he dine\". The predicted edit applies the same core change (\"why\" \u2192 \"Why\") and simply includes extra surrounding context (a preceding quotation mark and following \"to-\"). Since the essential capitalization correction is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the cold stones!\"",
"corrected_text": "the cold stones?\"",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation in the phrase \"cold stones!\" to \"cold stones?\". The prediction makes the same core change (exclamation point to question mark) within a slightly longer span (\"the cold stones!\\\"\" \u2192 \"the cold stones?\\\"\"), adding surrounding context (\"the\" and a closing quote). This still captures the essential intended edit."
}
],
"file_id": "004"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:36.149019",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:28.406401",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "\"No sir,",
"corrected_text": "\"No, sir,",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the types match exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"No\" in \"No sir\" \u2192 \"No, sir\". The prediction likewise inserts a comma after \"No\" in the broader context \"\\\"No sir,\" \u2192 \"\\\"No, sir,\". Although the prediction includes extra surrounding punctuation (an opening quotation mark and an existing trailing comma after \"sir\"), it correctly captures the core intended edit: adding the comma after \"No\"."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "\"why did he dine",
"corrected_text": "\"Why did he dine",
"observed_line_number": 30,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is capitalizing the initial \"why\" to \"Why\" (\"why did he dine\" \u2192 \"Why did he dine\"). The prediction captures the same core change while including an additional leading quotation mark in both the original and corrected text (\"\\\"why did he dine\" \u2192 \"\\\"Why did he dine\"). This extra context does not alter the essential capitalization correction, so the intention is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "the cold stones!\"",
"corrected_text": "the cold stones?\"",
"observed_line_number": 31,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation from an exclamation mark to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction applies the same core change (\"!\" \u2192 \"?\") within a slightly longer span (\"the cold stones!\\\"\" \u2192 \"the cold stones?\\\"\"). Including extra surrounding context (the word \"the\" and a closing quote) is acceptable, and the essential correction is captured accurately."
}
],
"file_id": "004"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:44.427434",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:02:46.901781",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "\"No sir,",
"corrected_text": "\"No, sir,",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma between \u201cNo\u201d and \u201csir\u201d (\"No sir\" \u2192 \"No, sir\"). The predicted edit likewise inserts a comma after \u201cNo\u201d (\"No sir,\" \u2192 \"No, sir,\"). Although the prediction includes extra surrounding context (an opening quote and an existing trailing comma), it still captures the core intended change: adding the comma after \u201cNo\u201d. Therefore, the essential punctuation correction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "\"why did he dine",
"corrected_text": "\"Why did he dine",
"observed_line_number": 30,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth changes the initial word from \"why\" to \"Why\" in the phrase \"why did he dine.\" The prediction applies the same core capitalization change (\"why\" \u2192 \"Why\") and simply includes an additional leading quotation mark in both the original and corrected text. This added context does not alter the essential intended correction, so the core change is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "the cold stones!\"",
"corrected_text": "the cold stones?\"",
"observed_line_number": 31,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation in \"cold stones!\" to \"cold stones?\" (exclamation to question mark). The prediction performs the same core change (! \u2192 ?) on the same phrase, merely including additional surrounding context (the preceding word \"the\" and a closing quote). Since the essential punctuation correction is captured, the text content matches the ground truth\u2019s intention."
}
],
"file_id": "004"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:34.780420",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:40.138356",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"if he\" to \"If he,\" capturing the same core capitalization change with identical original and corrected text. Therefore, the prediction correctly reflects the intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the verb \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The predicted edit captures this core change by replacing \"said\" with \"asked\". Although it omits the surrounding context word \"Clennam\" in the original/corrected_text fields, this does not alter the essential intended correction (said \u2192 asked), so it accurately reflects the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:30.597145",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:30.305931",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text \"if he\" and corrected text \"If he,\" capturing the identical core change (capitalizing the initial 'I'). Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both identify the same original text (\"said Clennam\") and the same corrected text (\"asked Clennam\"), capturing the identical core change with no discrepancies. Therefore, the predicted edit correctly matches the ground truth intention."
}
],
"file_id": "005"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:30.736663",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:02:38.507971",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:44.281299",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:27.032568",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road,\" capturing the identical core change (hyphenation). Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"materials\" to turn \"materials costing\" into \"materials, costing\". The core punctuation correction is captured identically.\nTherefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same core change: inserting a comma so that \"about and\" becomes \"about, and.\" The original and corrected text match the ground truth\u2019s intended punctuation change.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"association, did\") and the same corrected text (\"association did\"), capturing the core intended change: removal of the comma after \"association\". Line number differences are irrelevant per instructions. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 21,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly\". This captures the same core punctuation correction with identical original and corrected text.\n\nTherefore, ignoring line numbers, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction also adds a comma immediately after \"said\" within a larger context (\"always said when\" \u2192 \"always said, when\"). This captures the same core punctuation correction, just with additional surrounding words. Therefore, the essential edit matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said;",
"corrected_text": "and said,",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:48.791222",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:37.998260",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"high road\" to \"high-road,\" capturing the identical core punctuation change (inserting a hyphen). No discrepancy in original or corrected text content."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"materials\" to change \"materials costing\" to \"materials, costing\". The original and corrected text fields match the ground truth exactly.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"about\" to convert \"about and\" into \"about, and.\" The core punctuation correction is identical. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), inserting a comma after \"earnestly\". The core change is identical, so the prediction captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"said\" and \"when\" (\"said when\" \u2192 \"said, when\"). The predicted edit applies the same comma insertion in the same location, but includes extra surrounding context (\"always\" before the phrase). This added context does not alter the core intended correction. Therefore, the essential punctuation change is correctly captured."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The prediction performs the same core punctuation change\u2014adding a comma after \"Clennam\"\u2014but includes extra surrounding context (\"amazement\"). This additional context does not alter the essential edit, so it correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion, did not mend",
"corrected_text": "tion did not mend",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said;",
"corrected_text": "and said,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:49.219730",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:03:02.429145",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "broad high road",
"corrected_text": "broad high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to hyphenate \"high road\" \u2192 \"high-road\". The prediction applies the same core change, but includes extra surrounding context: \"broad high road\" \u2192 \"broad high-road\". This still captures the essential intended edit (inserting the hyphen between \"high\" and \"road\"), so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"materials costing\" to \"materials, costing\", correctly capturing the insertion of a comma after \"materials\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tossed about and",
"corrected_text": "tossed about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"about\" in the phrase \"about and\" \u2192 \"about, and\". The prediction applies the same core punctuation change, just with additional surrounding context: \"tossed about and\" \u2192 \"tossed about, and\". The essential edit (adding the comma after \"about\") is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly.\" The core punctuation change is identical with no discrepancies."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth edit inserts a comma after \"said\" in the phrase \"said when\" \u2192 \"said, when.\" The predicted edit does the same comma insertion after \"said\" but includes additional surrounding context (\"always\"). This added context does not change the core intended correction, which is the comma insertion after \"said.\" Therefore, the prediction captures the essential change."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction adds a comma after \"said\" as well, but includes extra preceding context (\"and said\" \u2192 \"and said,\"). This preserves the same core punctuation change\u2014placing a comma immediately after \"said\"\u2014so it correctly captures the intended edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth core change is inserting a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The predicted edit also inserts a comma after \"Clennam\" (\"Clennam in amazement\" \u2192 \"Clennam, in amazement\"). The prediction includes extra surrounding context (\"amazement\"), but the essential punctuation change is the same and correctly reflects the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion did not mend",
"corrected_text": "tion, did not mend",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:49.105455",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:19.489777",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I,",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:04:49.558668",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:26.720211",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to make the title end with a period (\"Sun and Shadow\" \u2192 \"Sun and Shadow.\"). The predicted edit also results in the title ending with a period (\"Sun and Shadow,\" \u2192 \"Sun and Shadow.\"). Although the predicted original_text includes a comma that is not present in the ground truth original_text, the core intended correction\u2014ensuring the title ends with a period\u2014is correctly captured."
}
],
"file_id": "007"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:08.776695",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:02:37.158194",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:19.975006",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:28.839379",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify changing \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The core change is identical, with matching original and corrected text. Therefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text \"dull autumn\" and the same corrected text \"dull, autumn\", capturing the identical core change (inserting a comma after \"dull\"). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"man\" (\"man slowly\" \u2192 \"man, slowly\"). The prediction applies the same core change\u2014adding a comma after \"man\"\u2014but includes additional surrounding context (\"One\"). This extra context does not alter the essential correction, and the intended punctuation change is captured correctly."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original segment \"Chalons was\" and the same corrected segment \"Chalons, was\", capturing the identical core change (inserting a comma after \"Chalons\"). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden with wet",
"corrected_text": "soddened with wet",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes \"sodden\" \u2192 \"soddened\". The predicted edit changes \"sodden with wet\" \u2192 \"soddened with wet\", which includes additional surrounding context but preserves the same core substitution (\"sodden\" \u2192 \"soddened\"). Therefore, the essential intended correction is correctly captured."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees against",
"corrected_text": "poplar-trees, against",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:23.918545",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:51.058361",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. This captures the exact core punctuation correction intended in the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit type matches exactly: both are labeled as \"punctuation\". Text content also matches the ground truth\u2019s core change: inserting a comma after \"dull\" to change \"dull autumn\" into \"dull, autumn\". No discrepancies in original or corrected text relevant to the intended edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"replacement\", so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the o). The original and corrected texts match the ground truth\u2019s intended change exactly.\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core change is identical with no discrepancies in original or corrected text. Therefore, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"man\" (\"man slowly\" \u2192 \"man, slowly\"). The predicted edit does the same core change, just with additional surrounding context (\"One man slowly\" \u2192 \"One man, slowly\"). The essential punctuation insertion is identical in intention and placement.\n\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change, inserting a comma after \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). The core punctuation correction is identical.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with wet",
"corrected_text": "soddened with wet",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:23.975944",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:02:58.094119",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same core change\u2014adding a period to \"Let Loose\" to make it \"Let Loose.\" The original and corrected texts match the ground truth\u2019s intention precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original phrase \"dull autumn\" and the same corrected phrase \"dull, autumn\", capturing the identical core change (insertion of a comma after \"dull\"). Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The original and corrected texts are identical between ground truth and prediction, so the core change is captured correctly."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core change is identical with no discrepancy in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma after \"man\" (\"man slowly\" \u2192 \"man, slowly\"). The prediction makes the same core punctuation change, just with additional preceding context (\"One man slowly\" \u2192 \"One man, slowly\"). The essential intended edit\u2014adding a comma after \"man\"\u2014is correctly captured."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change, inserting a comma after \"Chalons\" so that \"Chalons was\" becomes \"Chalons, was\". The core punctuation change is identical in original and corrected text. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with wet",
"corrected_text": "soddened with wet",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction are \"replacement\" edits, so the type matches exactly.\n2. Text Content Accuracy: The ground truth replaces \"sodden\" with \"soddened.\" The prediction replaces the longer span \"sodden with wet\" with \"soddened with wet,\" which includes extra surrounding context but preserves the same core change (\"sodden\" \u2192 \"soddened\"). Therefore, it correctly captures the intended correction."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar-trees against",
"corrected_text": "poplar-trees, against",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:17.939783",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:45.065640",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so much",
"corrected_text": "heart, so much",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth inserts a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The prediction applies the same core punctuation change, adding a comma after \"heart\", while including extra surrounding context (\"heart so much\" \u2192 \"heart, so much\"). This preserves the essential intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did",
"corrected_text": "(as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The core intended change is inserting the word \"indeed\" into the phrase \"as she did\" to make it \"as indeed she did.\" The predicted edit makes the same insertion, merely including an extra preceding parenthesis \"(\" as additional context in both the original and corrected strings. Since the essential modification (adding \"indeed\") is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction performs the same core punctuation insertion in the corresponding phrase, but includes additional surrounding context (\"at length said\" \u2192 \"at length, said\"). This extra word does not alter the essential intended edit, which is the comma after \"length.\""
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly. Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change (tense shift from hope \u2192 hoped). Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Good, generous Mr.",
"corrected_text": "Good, generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:20.723911",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:50.331309",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so much",
"corrected_text": "heart, so much",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth inserts a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The prediction applies the same core punctuation change but includes additional surrounding context (\"heart so much\" \u2192 \"heart, so much\"). This still captures the essential intended correction: adding a comma after \"heart\" before \"so\"."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did",
"corrected_text": "(as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the type matches.\n\n2) Text Content Accuracy: The core intended change is inserting the word \"indeed\" into the phrase \"as she did\" to make \"as indeed she did.\" The prediction captures this exact insertion. The additional leading parenthesis \"(\" included in both the predicted original_text and corrected_text is extra surrounding context and does not alter the essential edit. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction inserts the same comma in the same location, but includes additional surrounding context (\"at length said\" \u2192 \"at length, said\"). This still captures the core intended punctuation change.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change (tense shift from hope \u2192 hoped). Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Good, generous Mr.",
"corrected_text": "Good, generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:19.101632",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:02:57.850057",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"heart so\") and the same correction (\"heart, so\"), capturing the identical core change of inserting a comma after \"heart.\" Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she did",
"corrected_text": "(as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The core intended change is inserting the word \"indeed\" into the phrase \"as she did\" to make it \"as indeed she did.\" The prediction captures this same insertion, merely including an extra preceding parenthesis as additional context: \"(as she did\" \u2192 \"(as indeed she did.\" Since the essential modification (adding \"indeed\" after \"as\") is correctly represented, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction performs the same core punctuation change\u2014adding a comma after \"length\" before \"said\"\u2014but includes additional surrounding context (\"at length said\" \u2192 \"at length, said\"). This extra context does not alter the essential edit, which is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change from present to past tense. No discrepancies in content."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness)",
"corrected_text": "ness),",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:23.368805",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:35.265130",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "from hers and",
"corrected_text": "from hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" (\"hers and\" \u2192 \"hers, and\"). The prediction applies the same core punctuation change, only with additional preceding context (\"from hers and\" \u2192 \"from hers, and\"). The essential intended edit\u2014the comma insertion after \"hers\"\u2014is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters little",
"corrected_text": "on hers, matters little",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" in the phrase \"hers matters\" \u2192 \"hers, matters\". The predicted edit also inserts a comma after \"hers\" (\"on hers matters little\" \u2192 \"on hers, matters little\"). Although the prediction includes additional surrounding context, it captures the same core punctuation change at the same location. Therefore, the text content matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "for a moment with",
"corrected_text": "for a moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"moment\" (\"moment with\" \u2192 \"moment, with\"). The prediction performs the same core punctuation change\u2014adding a comma after \"moment\"\u2014but includes additional surrounding context (\"for a moment with\" \u2192 \"for a moment, with\"). Since the essential change is identical, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so beautiful that",
"corrected_text": "so beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The core intended change is inserting a comma after \"beautiful\" (i.e., changing \"beautiful that\" to \"beautiful, that\"). The prediction applies the same comma insertion in the same location, but includes additional surrounding context (\"so\"). Adding extra context is allowed as long as the essential punctuation change is captured, which it is."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\u201cthey are to be married.\u201d",
"corrected_text": "\u201cThey are to be married.\u201d",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the word \"they\" to \"They.\" The predicted edit performs the same core change, but includes additional surrounding context (the full quoted sentence). This still captures the essential intended correction: initial \"they\" \u2192 \"They.\" Therefore, the text content matches the ground truth\u2019s intention."
}
],
"file_id": "010"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:24.507949",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:42.415327",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "from hers and",
"corrected_text": "from hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after \"hers\" (\"hers and\" \u2192 \"hers, and\"). The prediction performs the same core change, adding the comma after \"hers\", but includes extra surrounding context (\"from hers and\" \u2192 \"from hers, and\"). This preserves the essential intended punctuation correction.\n\nBecause both the edit type and the core textual change match the ground truth\u2019s intention, the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters little",
"corrected_text": "on hers, matters little",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The predicted edit also inserts a comma after \"hers\" within a longer surrounding phrase (\"on hers matters little\" \u2192 \"on hers, matters little\"). Although it includes extra context words, it captures the same core punctuation change at the same location.\n\nTherefore, the predicted edit correctly reflects the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "a moment with a look",
"corrected_text": "a moment, with a look",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"moment\" (\"moment with\" \u2192 \"moment, with\"). The prediction applies the same core punctuation change within a longer span of text (\"a moment with a look\" \u2192 \"a moment, with a look\"). Including extra surrounding words is acceptable, and the essential intended edit\u2014adding a comma after \"moment\"\u2014is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so beautiful that it was well",
"corrected_text": "so beautiful, that it was well",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"beautiful\" and \"that\" (\"beautiful that\" \u2192 \"beautiful, that\"). The prediction applies the same core punctuation change\u2014adding a comma after \"beautiful\"\u2014but includes extra surrounding context (\"so beautiful that it was well\" \u2192 \"so beautiful, that it was well\"). This still captures the essential intended edit (comma insertion after \"beautiful\"), so the text content matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\u201cthey are to be married.\u201d",
"corrected_text": "\u201cThey are to be married.\u201d",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the word \"they\" to \"They.\" The predicted edit applies that same capitalization change, showing it within a larger quoted context (\"they are to be married.\" \u2192 \"They are to be married.\"). Including additional surrounding words is acceptable, and the core change (lowercase 'they' to capitalized 'They') is correctly captured."
}
],
"file_id": "010"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:25.452997",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:02:41.525863",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "from hers and",
"corrected_text": "from hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"hers\" in the phrase \"hers and\" \u2192 \"hers, and\". The predicted edit does the same core change but includes additional surrounding context: \"from hers and\" \u2192 \"from hers, and\". The essential punctuation insertion (comma after \"hers\") is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "on hers matters",
"corrected_text": "on hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The prediction also inserts a comma after \"hers\" within a longer phrase (\"on hers matters\" \u2192 \"on hers, matters\"). Although it includes additional surrounding context (the preceding word \"on\"), the core intended punctuation change is the same.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "a moment with",
"corrected_text": "a moment, with",
"observed_line_number": 26,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The core intended change is inserting a comma after the word \"moment\" in the phrase \"moment with\" \u2192 \"moment, with.\" The predicted edit does the same comma insertion, simply including an extra preceding word (\"a moment with\" \u2192 \"a moment, with\"). This added context does not alter the essential punctuation change and correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "so beautiful that",
"corrected_text": "so beautiful, that",
"observed_line_number": 28,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"beautiful\" in the phrase \"beautiful that\" \u2192 \"beautiful, that.\" The prediction likewise inserts a comma after \"beautiful\" (\"so beautiful that\" \u2192 \"so beautiful, that\"), merely including the additional preceding context word \"so.\" The essential punctuation change is the same and correctly captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "\u201cthey are to be married.\u201d",
"corrected_text": "\u201cThey are to be married.\u201d",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:30.357336",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:46.575055",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: removing the comma in \"that, in\" to make it \"that in\". The original and corrected text match the ground truth\u2019s intended punctuation removal with no deviation."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change\u2014removing the comma between \"fancy\" and \"whatever\" (\"fancy, whatever\" \u2192 \"fancy whatever\"). The corrected text matches the ground truth intention precisely."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation in the phrase from \"Thank you!\" to \"Thank you,\" (exclamation mark to comma). The prediction performs the same core change, just with additional surrounding context (quotation marks): \"Thank you!\" \u2192 \"Thank you,\". Since the essential punctuation change is identical and extra context is allowed, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once were",
"corrected_text": "than once were",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are a \"replacement,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is specifically \"that\" \u2192 \"than.\" The predicted edit expands the span to \"that once were\" \u2192 \"than once were,\" but the core intended correction (replacing \"that\" with \"than\") is preserved exactly, with only additional surrounding context included. This matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:27.743269",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:27.701476",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "\u201cThank you!\u201d",
"corrected_text": "\u201cThank you,\u201d",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation in \"Thank you!\" to \"Thank you,\" (exclamation mark replaced with a comma). The prediction performs the same core change, but includes surrounding quotation marks as additional context: \u201cThank you!\u201d \u2192 \u201cThank you,\u201d. This preserves the essential intended correction (replacing ! with ,), so the text content accurately matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core change: inserting a comma to change \"shore and\" to \"shore, and.\" The original and corrected text strings match the ground truth\u2019s intended punctuation edit. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "things that once",
"corrected_text": "things than once",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The core intended change in the ground truth is replacing the word \"that\" with \"than\". The predicted edit replaces the phrase \"things that once\" with \"things than once\", which includes the same essential substitution (\"that\" \u2192 \"than\") with additional surrounding context. Since the central correction is preserved, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in our fancy whatever",
"corrected_text": "that, in our fancy, whatever",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:49.459816",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:02:37.651789",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cThank you!\u201d",
"corrected_text": "\u201cThank you,\u201d",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation point to a comma (\"Thank you!\" \u2192 \"Thank you,\"). The prediction makes the same core change (! \u2192 ,) and merely includes additional surrounding context by retaining the quotation marks (\"\u201cThank you!\u201d\" \u2192 \"\u201cThank you,\u201d\"). This preserves the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"shore\" in the phrase \"shore and\" to make it \"shore, and.\" The original_text and corrected_text align with the ground truth\u2019s intended punctuation change. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "things that once",
"corrected_text": "things than once",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing the word \"that\" with \"than\". The prediction replaces the phrase \"things that once\" with \"things than once\", which preserves the same core substitution (\"that\" \u2192 \"than\") while including additional surrounding context. This correctly captures the intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that, in our fancy, whatever",
"corrected_text": "that in our fancy whatever",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:06:16.555208",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:58:49.762565",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to convert \"Chapter XXIX\" into \"Chapter XXIX.\". The original_text and corrected_text are identical between ground truth and prediction, capturing the intended punctuation correction precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "MRS. FLINTWINCH GOES ON DREAMING",
"corrected_text": "MRS. FLINTWINCH GOES ON DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period after the word \"DREAMING\" (\"DREAMING\" \u2192 \"DREAMING.\"). The predicted edit applies the same punctuation change at the end of a longer phrase ending in \"DREAMING\" (\"... DREAMING\" \u2192 \"... DREAMING.\"). This includes additional surrounding context but captures the same core correction: inserting a period immediately after \"DREAMING\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the city",
"corrected_text": "the City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is specifically \"city\" \u2192 \"City\". The prediction expands the span to include surrounding context (\"the city\" \u2192 \"the City\"), but the only substantive change is still the capitalization of \"city\" to \"City\". This captures the core intended edit accurately."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "reluctant return of",
"corrected_text": "reluctant return to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are of type \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth replaces the word \"of\" with \"to\". The predicted edit replaces the longer span \"reluctant return of\" with \"reluctant return to\", which includes additional surrounding context but preserves the same core correction (changing \"of\" to \"to\"). Therefore, the prediction captures the essential intended change."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes a comma after \"them\" (\"them,\") to a semicolon (\"them;\"). The predicted edit applies the same punctuation change (comma to semicolon) at the same point, merely including extra surrounding context (\"familiar with\"). This still captures the core intended correction.\n\nTherefore, the predicted edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence at",
"corrected_text": "busy existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change within a longer span of surrounding context (\"busy existence at\" \u2192 \"busy existence, at\"), which still places the comma immediately after \"existence\". The additional words included do not alter the essential intended punctuation correction.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it, to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes a comma to a semicolon after the word \"it\" (\"it,\" \u2192 \"it;\"). The predicted edit also changes the comma to a semicolon in the same location, merely including extra surrounding context (\"it, to suppose\" \u2192 \"it; to suppose\"). The core punctuation change is identical, so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change, adding a comma after \"motionless\" in the slightly larger context of the following word (\"motionless when\" \u2192 \"motionless, when\"). This captures the essential intended punctuation insertion.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "a stand-still, to",
"corrected_text": "a stand-still; to",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2. Text Content Accuracy: The ground truth changes the punctuation after \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The prediction makes the same core change, simply including additional surrounding context (\"a \" before and \" to\" after): \"a stand-still,\" \u2192 \"a stand-still; to\". This captures the essential intended correction."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change in a longer snippet (\"our view by\" \u2192 \"our view, by\"), which still clearly captures inserting a comma immediately after \"view\". The added surrounding context does not alter the essential edit.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 10,
"expected_edit_num": 10,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "contracted existence, is",
"corrected_text": "contracted existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after the word \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction applies the same core change within a larger surrounding span (\"contracted existence, is\" \u2192 \"contracted existence; is\"). Although it includes additional context words, the essential intended correction\u2014replacing the comma after \"existence\" with a semicolon\u2014is correctly captured."
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:06:12.126776",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:00:41.198805",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Chapter XXIX\") and the same corrected text (\"Chapter XXIX.\"), capturing the core intended change of adding a period. Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the city",
"corrected_text": "the City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"capitalization,\" which matches the ground truth exactly.\n2) Text Content Accuracy: The ground truth changes \"city\" to \"City\". The prediction changes \"the city\" to \"the City,\" which includes extra surrounding context but preserves the same core capitalization change on the word \"city.\" Therefore it captures the intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "reluctant return of",
"corrected_text": "reluctant return to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces the word \"of\" with \"to\". The predicted edit replaces the phrase \"reluctant return of\" with \"reluctant return to\", which includes extra surrounding context but preserves the same core change: \"of\" \u2192 \"to\". Therefore, it correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"them\" from a comma to a semicolon (\"them,\" \u2192 \"them;\"). The predicted edit applies the same core change, but includes extra surrounding context (\"familiar with them,\" \u2192 \"familiar with them;\"). This still captures the essential intended correction: replacing the comma with a semicolon after \"them\".\n\nTherefore, the predicted edit correctly captures the ground truth\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence at",
"corrected_text": "busy existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit adds a comma after \"existence\" within a longer phrase (\"busy existence at\" \u2192 \"busy existence, at\"). This includes additional surrounding context but captures the same core punctuation change at the same word boundary.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change the same original text \"it,\" to the same corrected text \"it;\". This captures the core intended punctuation change (comma to semicolon) with no discrepancy."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit applies the same core change within a slightly larger context (\"motionless when\" \u2192 \"motionless, when\"), which still places the comma after \"motionless.\" This captures the essential intended punctuation correction."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "a stand-still,",
"corrected_text": "a stand-still;",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The predicted edit performs the same core change, just with additional left context (\"a stand-still,\" \u2192 \"a stand-still;\"). Including extra surrounding words is acceptable as long as the essential punctuation change is captured, which it is.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction adds the same comma after \"view\" within a larger context (\"our view by\" \u2192 \"our view, by\"). This captures the same core punctuation insertion intended by the ground truth, just with extra surrounding words included.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "contracted existence,",
"corrected_text": "contracted existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation immediately following \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction applies the same core punctuation change at the same point, but includes additional surrounding context (\"contracted existence,\" \u2192 \"contracted existence;\"). Including extra preceding words is allowed as long as the essential change is the same, which it is here (comma \u2192 semicolon after \"existence\")."
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:06:18.140146",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:02:58.772699",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Chapter XXIX\" to \"Chapter XXIX.\" by adding a period at the end. The core intended punctuation change is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "MRS. FLINTWINCH GOES ON DREAMING",
"corrected_text": "MRS. FLINTWINCH GOES ON DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a period to \"DREAMING\" \u2192 \"DREAMING.\". The prediction adds a period at the end of a longer string ending in the same target word: \"MRS. FLINTWINCH GOES ON DREAMING\" \u2192 \"MRS. FLINTWINCH GOES ON DREAMING.\". This captures the same core intended correction (adding a final period after DREAMING), merely with additional surrounding context."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the city",
"corrected_text": "the City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth change is capitalizing \"city\" to \"City\". The prediction applies the same capitalization change within a slightly larger context (\"the city\" \u2192 \"the City\"). This preserves the core intended edit (capitalizing the word \"city\"). Therefore, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "reluctant return of",
"corrected_text": "reluctant return to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing the word \"of\" with \"to\". The predicted edit replaces the phrase \"reluctant return of\" with \"reluctant return to\", which includes additional surrounding context but preserves the essential substitution of \"of\" \u2192 \"to\". This correctly captures the core intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "familiar with them,",
"corrected_text": "familiar with them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"them\" from a comma to a semicolon (\"them,\" \u2192 \"them;\"). The prediction performs the same core change, only with additional surrounding context (\"familiar with them,\" \u2192 \"familiar with them;\"). Since the essential punctuation change is identical, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence at",
"corrected_text": "busy existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit adds a comma after \"existence\" within a longer span (\"busy existence at\" \u2192 \"busy existence, at\"). This captures the same core punctuation change, merely with additional surrounding context.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it, to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The core intended change is replacing a comma after \"it\" with a semicolon (\"it,\" \u2192 \"it;\"). The predicted edit performs the same comma-to-semicolon substitution, merely including additional surrounding context (\"to suppose\"). This still captures the essential punctuation change identified in the ground truth."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma immediately after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core punctuation change, adding a comma after \"motionless\" while including extra surrounding context (\"motionless when\" \u2192 \"motionless, when\"). This preserves the essential intended edit.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "a stand-still, to",
"corrected_text": "a stand-still; to",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The prediction performs the same core change, merely including extra surrounding context (\"a stand-still, to\" \u2192 \"a stand-still; to\"). The essential correction (comma \u2192 semicolon after \"stand-still\") is correctly captured."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "our view by",
"corrected_text": "our view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction expands the span to include surrounding context (\"our view by\" \u2192 \"our view, by\"), but the core change is the same: a comma is added immediately after \"view\". The extra context does not alter the essential intended correction.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 10,
"expected_edit_num": 10,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "contracted existence, is",
"corrected_text": "contracted existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is specifically replacing the comma after \"existence,\" with a semicolon (\"existence; \"). The predicted edit performs the same core punctuation substitution at the same point, merely including additional surrounding context (\"contracted\" before and \"is\" after). This still captures the essential intended correction."
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:06:02.688889",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:59:23.652541",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid it",
"corrected_text": "now, I have paid it",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The predicted edit also inserts a comma after \"now\" within a longer phrase (\"now I have paid it\" \u2192 \"now, I have paid it\"). This includes extra surrounding context but preserves the same core punctuation change. Therefore, it correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\nText Content Accuracy: Both replace the same original word \"authorize\" with the same corrected word \"authorise\". The core intended change (American to British spelling) is captured exactly with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam;",
"corrected_text": "said Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Your own recompense",
"corrected_text": "Your own recompense",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to break",
"corrected_text": "now, to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that-boiling-over",
"corrected_text": "that boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty;",
"corrected_text": "at twenty,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "out of him, than",
"corrected_text": "out of him than",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:06:10.257131",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:01:39.762683",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\nText Content Accuracy: Both target the same word \"him\" with no change in text content other than applying italics. The prediction captures the same core intended change (italicizing \"him\").\nTherefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid it",
"corrected_text": "now, I have paid it",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The core intended change in the ground truth is inserting a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The predicted edit also inserts a comma after \"now\" (\"now I have paid it\" \u2192 \"now, I have paid it\"). Although the prediction includes additional surrounding words, it captures the same essential punctuation correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the original text \"authorize\" with the corrected text \"authorise\". The core intended change (American to British spelling) is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam;",
"corrected_text": "said Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to break",
"corrected_text": "now, to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that-boiling-over",
"corrected_text": "that boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty;",
"corrected_text": "at twenty,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:06:07.942147",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:03:47.916495",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "out of him",
"corrected_text": "out of *him*",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates that the word \"him\" should be italicized (with no change to the surrounding text). The prediction expands the scope to include surrounding context (\"out of him\" \u2192 \"out of *him*\"), but the core change still correctly italicizes only the word \"him\". Including extra surrounding words as context is acceptable, and it does not alter the intended core edit.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid it",
"corrected_text": "now, I have paid it",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The predicted edit also inserts a comma after \"now\" and includes extra surrounding context (\"now I have paid it\" \u2192 \"now, I have paid it\"). This preserves the same core punctuation change and correctly captures the intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both replace the same original word \"authorize\" with the same corrected form \"authorise\". The core intended change (American to British spelling) is captured identically. Therefore, the predicted edit matches the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam;",
"corrected_text": "said Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to break",
"corrected_text": "now, to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that-boiling-over",
"corrected_text": "that boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty;",
"corrected_text": "at twenty,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:53.472954",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T03:59:09.154763",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit have the type \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text \"For Mr. Pancks\" and the same corrected text \"For, Mr. Pancks\", capturing the core intended change (inserting a comma after \"For\"). Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so quickly that",
"corrected_text": "so quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"quickly\" in the sequence \"quickly that\" \u2192 \"quickly, that.\" The prediction likewise inserts a comma after \"quickly\" (\"so quickly that\" \u2192 \"so quickly, that\"), adding extra preceding context (\"so\") but preserving the same core punctuation change at the same point. This correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "making me not",
"corrected_text": "making me a not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:05:56.251299",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:01:13.128018",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same correction (inserting a comma after \"For\" to make \"For, Mr. Pancks\"). The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so quickly that",
"corrected_text": "so quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"quickly\" in the phrase \"quickly that\" \u2192 \"quickly, that\". The prediction applies the same core change (comma after \"quickly\") but includes additional surrounding context: \"so quickly that\" \u2192 \"so quickly, that\". This preserves the essential intended punctuation correction, so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not that",
"corrected_text": "making me, not that",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_claude_opus_5",
"date": "2026-09-06T04:06:00.242724",
"annotator": "Claude Opus 5 (OR)",
"annotation_date": "2026-09-06T04:03:25.793989",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same correction (inserting a comma after \"For\" to make \"For, Mr. Pancks\"). The core punctuation change is captured precisely with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so quickly that",
"corrected_text": "so quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"quickly\" (\"quickly that\" \u2192 \"quickly, that\"). The prediction performs the same core change\u2014adding a comma after \"quickly\"\u2014but includes extra surrounding context (\"so quickly that\" \u2192 \"so quickly, that\"). Including additional context is acceptable as long as the essential punctuation change is the same, which it is.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "making me, not",
"corrected_text": "making me a, not",
"observed_line_number": 21,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts the word \"a\" after \"me\" (\"me,\" \u2192 \"me a,\"). The predicted edit also inserts \"a\" in the same location within a larger context string (\"making me, not\" \u2192 \"making me a, not\"). Including extra surrounding words is acceptable, and the core change\u2014adding \"a\" after \"me\"\u2014is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2026-07-24",
"release_source": "https://www.anthropic.com/news/claude-opus-5",
"release_notes": "Fifth-generation Opus model. Benchmark configuration uses high reasoning effort.",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Sonnet 4.5",
"model_id": "claude-sonnet-4.5",
"date": "2026-03-08T11:24:47.461208",
"shots": 2,
"config": {
"model_id": "claude-sonnet-4.5",
"display_name": "Claude Sonnet 4.5",
"shots": 2,
"temperature": 0.0,
"date": "2026-03-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:51:43.868643",
"file_results": [
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:24:39.482517",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:51:43.868643",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over yonder to-day",
"corrected_text": "howling, over yonder, to-day",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day sir",
"corrected_text": "to-day, sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:13:21.286007",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:35:34.523003",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over yonder to-day",
"corrected_text": "howling, over yonder, to-day",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day sir",
"corrected_text": "to-day, sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:13:48.329720",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:54:18.892143",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over yonder to-day",
"corrected_text": "howling over yonder, to-day,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:25:07.522211",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:52:19.922926",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones?",
"corrected_text": "stones!",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "knew,",
"corrected_text": "knew",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at;",
"corrected_text": "at,",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir?",
"corrected_text": "sir",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:14:14.253862",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:36:09.457146",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones?",
"corrected_text": "stones!",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "knew,",
"corrected_text": "knew",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at;",
"corrected_text": "at,",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir?",
"corrected_text": "sir",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:14:42.188777",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:54:55.578090",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones?",
"corrected_text": "stones!",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "knew,",
"corrected_text": "knew",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at;",
"corrected_text": "at,",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir?",
"corrected_text": "sir",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:25:39.366190",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:52:54.924174",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 42,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces \u201csaid\u201d with \u201casked\u201d in the phrase \u201csaid Clennam\u201d \u2192 \u201casked Clennam.\u201d The prediction replaces \u201csaid\u201d \u2192 \u201casked\u201d without including the following context \u201cClennam,\u201d but it captures the essential intended change (the word substitution from \u201csaid\u201d to \u201casked\u201d). Additional surrounding context is optional, so this still matches the core correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "If he",
"corrected_text": "If he cap",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:15:11.915430",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:36:43.694192",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 42,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces the word \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the same core substitution \"said\" \u2192 \"asked\" but with less surrounding context (it omits \"Clennam\"). Since the essential intended change is the replacement of \"said\" with \"asked\", the prediction matches the ground truth\u2019s intent."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "If he",
"corrected_text": "If he cap",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:15:45.277168",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:55:30.778493",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 42,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n2. Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" within the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction replaces \"said\" with \"asked\" (without including the following context \"Clennam\"), but it captures the core intended change. Extra surrounding context is optional, so this matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "If he",
"corrected_text": "If he cap",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:26:18.506180",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:53:31.712694",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"said\" in the phrase \"said when\" \u2192 \"said, when\". The prediction captures this core change by editing \"said\" \u2192 \"said,\". Although it does not include the following word \"when\" in its original/corrected text span, it still reflects the essential intended punctuation insertion (a comma immediately after \"said\"). Therefore, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"Clennam in\" to \"Clennam, in\" by inserting a comma after \"Clennam\". The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said, (comma)",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:16:20.401147",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:37:19.666983",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"said\" in the sequence \"said when\" \u2192 \"said, when\". The predicted edit \"said\" \u2192 \"said,\" captures the same core punctuation change (adding a comma immediately after \"said\"). Although it omits the following word \"when\", it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"Clennam in\" to \"Clennam, in\", correctly inserting a comma after \"Clennam\". The core intended punctuation correction is captured exactly."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said, (comma)",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:16:55.039677",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:56:08.158464",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after the word \"said\" in the phrase \"said when\" \u2192 \"said, when\". The predicted edit \"said\" \u2192 \"said,\" captures this same core punctuation change (adding a comma immediately after \"said\"). Although it omits the following word \"when\" from the original/corrected strings, it still represents the essential intended correction: inserting the comma after \"said\" before what follows."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"Clennam in\" to \"Clennam, in,\" inserting a comma after \"Clennam.\" The core punctuation correction is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said, (comma)",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:12:40.484914",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:54:05.924093",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I,",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:17:15.951392",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:37:53.849420",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I,",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:17:38.815974",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:56:42.554546",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I,",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW,",
"corrected_text": "SUN AND SHADOW.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:26:57.544398",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:54:47.818212",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heavily; and",
"corrected_text": "heavily, and",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "poplar-trees",
"corrected_text": "poplar trees",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "solitary; and",
"corrected_text": "solitary, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hand; miry, footsore,",
"corrected_text": "hand, miry, footsore,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "untrimmed; the",
"corrected_text": "untrimmed, the",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "difficulty; he",
"corrected_text": "difficulty, he",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shrinkingly;",
"corrected_text": "shrinkingly,",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sa\u00f4ne",
"corrected_text": "Sa\u00f4ne.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar trees hyphen",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons,was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with wet; limping",
"corrected_text": "sodden with wet, limping",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:18:12.285918",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:38:39.002456",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heavily; and",
"corrected_text": "heavily, and",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "solitary; and",
"corrected_text": "solitary, and",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons, was",
"corrected_text": "Chalons was",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hand; miry, footsore,",
"corrected_text": "hand, miry, footsore,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "untrimmed; the",
"corrected_text": "untrimmed, the",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "difficulty; he",
"corrected_text": "difficulty, he",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shrinkingly;",
"corrected_text": "shrinkingly,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "muttering.",
"corrected_text": "muttering:",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sa\u00f4ne",
"corrected_text": "Sa\u00f4ne.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar trees, hyphen",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with wet; limping",
"corrected_text": "sodden with wet, limping",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:18:46.207011",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:57:22.868544",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heavily; and",
"corrected_text": "heavily, and",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees, hyphen",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "solitary; and",
"corrected_text": "solitary, and",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons, was",
"corrected_text": "Chalons was",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hand; miry, footsore,",
"corrected_text": "hand, miry, footsore,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "untrimmed; the",
"corrected_text": "untrimmed, the",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "difficulty; he",
"corrected_text": "difficulty, he",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shrinkingly;",
"corrected_text": "shrinkingly,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "muttering.",
"corrected_text": "muttering:",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sa\u00f4ne",
"corrected_text": "Sa\u00f4ne.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with wet; limping",
"corrected_text": "sodden with wet, limping",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:27:26.500395",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:55:22.174350",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"heart so\" to \"heart, so\", inserting a comma after \"heart\". The core intended punctuation correction is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr",
"corrected_text": "generous, Mr",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hope so",
"corrected_text": "hope so;",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (as she did, if it be the source of earnest-",
"corrected_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (indeed, as she did, if it be the source of earnest-",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:19:17.805766",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:39:17.852086",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"heart so\" to \"heart, so,\" inserting a comma after \"heart.\" The core punctuation change is captured exactly with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "as",
"corrected_text": "(as",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestness)",
"corrected_text": "earnestness),",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hoped so",
"corrected_text": "I hope so",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hope so;",
"corrected_text": "hope so:",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "tially",
"corrected_text": "indeed",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:19:49.910921",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:58:01.443567",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"heart\" in the phrase \"heart so\" to make it \"heart, so\". The original and corrected text fields are identical between ground truth and prediction, capturing the core intended punctuation change."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr",
"corrected_text": "generous, Mr",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hope so",
"corrected_text": "hope so;",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (as she did, if it be the source of earnest-",
"corrected_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (indeed, as she did, if it be the source of earnest-",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:27:51.009445",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:56:04.997445",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "here and",
"corrected_text": "here, and",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:20:13.858947",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:39:54.571059",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "here and",
"corrected_text": "here, and",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "this, he",
"corrected_text": "this, He",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly, \"they",
"corrected_text": "directly, \"They",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:20:41.444590",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:58:39.360662",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "here and",
"corrected_text": "here, and",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "At this, he thought directly, \"they are to be married.\"",
"corrected_text": "At this, he thought directly, \"They are to be married.\"",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:28:23.317199",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:56:44.395311",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Ignoring line numbers as instructed, the prediction fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"within doors\" to \"within-doors\". The prediction makes the same core change from \"within doors\" to \"within-doors\" with no discrepancy in the original or corrected text. Line numbers are to be ignored, so the differing line number does not affect correctness.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "I feel",
"corrected_text": "I feel",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you\"",
"corrected_text": "\"Thank you!\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "moonlight,",
"corrected_text": "moonlight, then",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Thank you,\"",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:21:19.853290",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:40:32.587662",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"shore\" in the phrase \"shore and\" to make it \"shore, and\". The original_text and corrected_text match the ground truth\u2019s intended modification. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: Both edits replace \"within doors\" with \"within-doors\". The core intended change\u2014hyphenating the phrase\u2014is captured identically in the prediction. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "I feel",
"corrected_text": "I feel",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you\"",
"corrected_text": "\"Thank you!\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "moonlight,",
"corrected_text": "moonlight, then",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Thank you,\"",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:21:52.312534",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:59:16.127413",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"shore\" in the phrase \"shore and\" to make it \"shore, and\". The original_text and corrected_text are identical between ground truth and prediction, so the intended punctuation correction is fully captured. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: Both edits replace the same original text \"within doors\" with the same corrected text \"within-doors\", capturing the identical core change (hyphenation). Any line number discrepancy is to be ignored per instructions and does not affect the correctness of the edit content."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the word \"that\" with \"than\". This captures the same core textual change with identical original and corrected text. Line numbers are ignored as instructed, so the mismatch there does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "I feel",
"corrected_text": "I feel",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you\"",
"corrected_text": "\"Thank you!\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Thank you,\"",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:28:52.556231",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:57:24.469582",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "these",
"corrected_text": "these,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "her",
"corrected_text": "her;",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "concerned",
"corrected_text": "concerned,",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege",
"corrected_text": "liege-",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wonder",
"corrected_text": "wonder,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:22:20.940380",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:41:24.791608",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "luctant return",
"corrected_text": "reluctant return",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "there",
"corrected_text": "there,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "her but",
"corrected_text": "her; but",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "city",
"corrected_text": "city (cap)",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:22:46.930809",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:59:55.470130",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "luctant return",
"corrected_text": "reluctant return",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with them",
"corrected_text": "with them;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "there",
"corrected_text": "there,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "when",
"corrected_text": "when,",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "her but",
"corrected_text": "her; but",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "city",
"corrected_text": "city (cap)",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:29:30.192707",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:58:03.917484",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest snorts\" to \"driest of snorts\" (fixing the spelling \"dryest\"\u2192\"driest\" and inserting \"of\" between \"driest\" and \"snorts\"). The predicted edit makes the same core change: \"dryest snorts\" \u2192 \"driest of snorts\". Although the ground truth includes surrounding context (\"of the\" before and a period after), the prediction is allowed to include less context as long as the essential change is captured, which it is.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both replace the same original text \"authorize\" with the same corrected text \"authorise\", capturing the identical core spelling change. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "boiling-over",
"corrected_text": "boiling over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "recompense",
"corrected_text": "recompense",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:23:26.218356",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:42:01.490408",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts an additional \"of\" to make the phrase \"driest of snorts.\" The predicted edit makes the same core change: \"dryest snorts\" \u2192 \"driest of snorts.\" Although the prediction omits surrounding context (\"of the\" at the start) and the final period, it still captures the essential correction (spelling change + insertion of \"of\"). Therefore, the predicted edit matches the intention of the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both identify the same original text (\"authorize\") and the same corrected text (\"authorise\"), capturing the identical core change (American to British spelling). Line numbers are to be ignored, so the line mismatch does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that boiling-over",
"corrected_text": "that-boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "recompense",
"corrected_text": "recompense",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:23:59.841540",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T11:00:30.975768",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts an additional \"of\" so the phrase becomes \"driest of snorts.\" The predicted edit likewise changes \"dryest\" \u2192 \"driest\" and includes the inserted \"of\" before \"snorts\" (\"driest of snorts\"). Although the prediction omits surrounding context (\"of the\" and the final period), it still captures the core intended change accurately."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 36,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both change the same original word \"authorize\" to the same corrected spelling \"authorise\". This captures the core intended correction (American to British spelling). Line numbers are to be ignored and do not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "boiling-over",
"corrected_text": "boiling over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "recompense",
"corrected_text": "recompense",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T10:29:57.002332",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T09:58:51.320814",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed. For Mr. Pancks",
"corrected_text": "bed. For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction applies the same core punctuation change, but includes extra surrounding context (\"bed. For Mr. Pancks\" \u2192 \"bed. For, Mr. Pancks\"). This still captures the essential intended edit\u2014adding the comma after \"For\"\u2014so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "morning; and",
"corrected_text": "morning; and,",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mr. Rugg",
"corrected_text": "Mr. Rugg,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "back; and",
"corrected_text": "back; and,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "papers, and",
"corrected_text": "papers, and,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam, he",
"corrected_text": "Clennam, he",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "down-stairs, and",
"corrected_text": "down-stairs, and",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "course, resolved",
"corrected_text": "course, resolved",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time; but",
"corrected_text": "time; but",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "street, and",
"corrected_text": "street, and",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "come, and",
"corrected_text": "come, and",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "herself, but",
"corrected_text": "herself, but",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was, and",
"corrected_text": "was, and",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "lady, \"who",
"corrected_text": "lady, \"who",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and pray excuse",
"corrected_text": "and pray excuse",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 15,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me; not",
"corrected_text": "me; not",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 16,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur, \"for",
"corrected_text": "Arthur, \"for",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 17,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "visit, but",
"corrected_text": "visit, but",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 18,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur,\" returned",
"corrected_text": "Arthur,\" returned",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 19,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Finching, \"pray",
"corrected_text": "Finching, \"pray",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 20,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view, at",
"corrected_text": "view, at",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 21,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view, but",
"corrected_text": "view, but",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 22,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tenderly, and",
"corrected_text": "tenderly, and",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 23,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time, but",
"corrected_text": "time, but",
"observed_line_number": 45,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 24,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly, that",
"corrected_text": "quickly, that",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:24:27.575707",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T10:42:49.968467",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed. For Mr. Pancks",
"corrected_text": "bed. For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after \"For\" in \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The prediction applies the same core change, showing \"... For Mr. Pancks\" \u2192 \"... For, Mr. Pancks\" (with extra preceding context \"bed.\"). Including additional surrounding words is allowed, and the essential punctuation insertion is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mr. Rugg",
"corrected_text": "Mr. Rugg,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "back; and",
"corrected_text": "back, and",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "papers, and",
"corrected_text": "papers and",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam, he",
"corrected_text": "Clennam he",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "down-stairs, and",
"corrected_text": "down-stairs and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly, that",
"corrected_text": "quickly that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time; but",
"corrected_text": "time, but",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "street, and",
"corrected_text": "street and",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "knocker, he",
"corrected_text": "knocker he",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "come, and",
"corrected_text": "come and",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "herself, but",
"corrected_text": "herself but",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was, and",
"corrected_text": "was and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "lady, \"who",
"corrected_text": "lady \"who",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pray excuse",
"corrected_text": "pray, excuse",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 15,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me; not",
"corrected_text": "me, not",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 16,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur,\"",
"corrected_text": "Arthur",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 17,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "visit, but",
"corrected_text": "visit but",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 18,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur,\"",
"corrected_text": "Arthur",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 19,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Finching, \"pray",
"corrected_text": "Finching \"pray",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 20,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "un-questionably",
"corrected_text": "unquestionably",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 21,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "still 'tis",
"corrected_text": "still, 'tis",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 22,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view, at",
"corrected_text": "view at",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 23,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "con-siderably",
"corrected_text": "considerably",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 24,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view, but",
"corrected_text": "view but",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 25,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tenderly, and",
"corrected_text": "tenderly and",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 26,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time, but",
"corrected_text": "time but",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-sonnet-4.5",
"date": "2026-03-08T11:24:47.461208",
"annotator": "Claude Sonnet 4.5",
"annotation_date": "2026-03-08T11:01:16.259508",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed. For Mr. Pancks",
"corrected_text": "bed. For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction captures the same core punctuation insertion, just with additional preceding context (\"bed. \"). Since the essential edit\u2014adding the comma after \"For\"\u2014is correctly represented, the text content matches the ground truth's intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "morning; and",
"corrected_text": "morning; and,",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mr. Rugg",
"corrected_text": "Mr. Rugg,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "back; and",
"corrected_text": "back; and,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "papers, and",
"corrected_text": "papers, and,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "down-stairs, and",
"corrected_text": "down-stairs, and,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time; but",
"corrected_text": "time; but,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "street, and",
"corrected_text": "street, and,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was, and",
"corrected_text": "was, and,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me; not",
"corrected_text": "me, not",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "visit, but",
"corrected_text": "visit; but",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur,",
"corrected_text": "Arthur;",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "still 'tis",
"corrected_text": "still, 'tis",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view, but",
"corrected_text": "view; but",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "again and",
"corrected_text": "again, and",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 15,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tenderly, and",
"corrected_text": "tenderly, and,",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 16,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "time, but",
"corrected_text": "time; but",
"observed_line_number": 44,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 17,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "past and",
"corrected_text": "past, and",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 18,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "come, and",
"corrected_text": "come, and,",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2025-09-29",
"release_source": "https://www.anthropic.com/claude/sonnet",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Sonnet 4.6",
"model_id": "claude-sonnet-4.6",
"date": "2026-03-08T11:25:31.262735",
"shots": 2,
"config": {
"model_id": "claude-sonnet-4.6",
"display_name": "Claude Sonnet 4.6",
"shots": 2,
"temperature": 0.0,
"date": "2026-03-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T09:59:32.091237",
"file_results": [
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:24:40.487448",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T09:59:32.091237",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have the type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change: adding a period to \"Fellow Travellers\" to make it \"Fellow Travellers.\". The original_text and corrected_text match the ground truth\u2019s intended punctuation correction precisely. Therefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"howling over\" to \"howling, over\" by inserting a comma after \"howling\". The original and corrected text match exactly, capturing the same core punctuation change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), inserting a comma after \"yonder\". This captures the core intended punctuation change with no discrepancy."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same corrected text (\"pockets, and\"), capturing the identical core change: inserting a comma after \"pockets\". Line number differences are ignored as instructed.\n\nTherefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:13:21.923379",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:43:39.993293",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have the type set to \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. This captures the same core punctuation correction with identical original and corrected text. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"yonder to-day\" to \"yonder, to-day\", inserting a comma after \"yonder\". The core punctuation correction is identical."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same corrected text (\"pockets, and\"), capturing the core intended change of inserting a comma after \"pockets\". Any difference in line number is to be ignored, and the textual edit itself matches perfectly."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:13:55.838020",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:01:56.446750",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"howling over\" \u2192 \"howling, over\", inserting a comma after \"howling\". The core intended change is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"yonder to-day\" to \"yonder, to-day\" by inserting a comma after \"yonder\". The original and corrected texts match the ground truth\u2019s intended punctuation change with no discrepancy."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to make it \"pockets, and\". The original_text and corrected_text match the ground truth\u2019s intended punctuation change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:11:20.345149",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:00:03.179505",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:14:21.795539",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:44:15.041743",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:14:47.949240",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:02:25.562113",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:11:44.978163",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:00:41.970232",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 48,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 51,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:15:15.789764",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:44:49.018880",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 48,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 51,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:15:42.227376",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:02:58.858899",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 53,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 56,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:25:10.591501",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:01:25.099928",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both the ground truth and prediction change \"materials costing\" to \"materials, costing\", inserting a comma after \"materials\". This captures the core intended punctuation correction with identical original and corrected text content.\n\nLine numbers are ignored as instructed, and no other discrepancies affect the essential edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion did",
"corrected_text": "tion, did",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said,",
"corrected_text": "said,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 42,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:16:10.356488",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:45:25.566190",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change: insertion of a comma after \"materials\". \n\nLine numbers are ignored as instructed, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion did",
"corrected_text": "tion, did",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said,",
"corrected_text": "said,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 42,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:16:38.003446",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:03:38.795791",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both identify the same original text (\"materials costing\") and apply the same correction by inserting a comma after \"materials\" to produce \"materials, costing.\" This captures the core intended change precisely. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion did",
"corrected_text": "tion, did",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 42,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:25:40.953433",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:02:01.139659",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to make the title end with a period: \"Sun and Shadow\" \u2192 \"Sun and Shadow.\". The prediction also results in the same corrected text, \"Sun and Shadow.\", capturing the core intended punctuation change (ensuring a period at the end). Although the predicted \"original_text\" includes a comma (\"Sun and Shadow,\") rather than no punctuation, the essential intended correction\u2014ending the phrase with a period\u2014is still correctly represented."
}
],
"file_id": "007"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:17:08.973584",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:45:58.285617",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intent is to make the title end with a period: \"Sun and Shadow\" \u2192 \"Sun and Shadow.\" The predicted edit also results in the same corrected text, \"Sun and Shadow.\" Although the prediction\u2019s \"original_text\" includes a comma (\"Sun and Shadow,\") rather than no punctuation, it still captures the core intended change: the ending punctuation should be a period. Therefore, the essential correction is represented correctly."
}
],
"file_id": "007"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:17:34.953321",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:04:13.387135",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:26:24.446774",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:02:39.780491",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"man slowly\" to \"man, slowly\", inserting a comma after \"man\". The core correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core change\u2014adding a comma after \"Chalons\" in the phrase \"Chalons was\" to make it \"Chalons, was.\" There is no mismatch in original or corrected text. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the same original text \"sodden\" with the same corrected text \"soddened\", capturing the core intended change precisely. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose,",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:18:15.990019",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:46:34.287778",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Let Loose\" to make it \"Let Loose.\". The original_text and corrected_text match the ground truth exactly, capturing the intended punctuation edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth inserts a comma after \"Chalons\" in the phrase \"Chalons was\" \u2192 \"Chalons, was\". The prediction similarly represents inserting a comma after \"Chalons\" by changing \"Chalons\" \u2192 \"Chalons,\". Although the prediction uses less surrounding context (it does not include the following word \"was\"), it captures the same core intended change: adding a comma immediately after \"Chalons\"."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the original text \"sodden\" with the corrected text \"soddened\". The core intended change is captured identically with no discrepancy. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:18:54.152662",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:04:50.619669",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change\u2014adding a period to the end of \"Let Loose\" to make it \"Let Loose.\" The original and corrected texts are identical between ground truth and prediction, capturing the core intended punctuation change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth inserts a comma after the word \"Chalons\" (changing \"Chalons was\" to \"Chalons, was\"). The prediction also captures this same core change by changing \"Chalons\" to \"Chalons,\". Although the prediction includes less surrounding context (it omits \"was\"), it still correctly represents the essential intended punctuation insertion at the same location."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: Both edits replace \"sodden\" with \"soddened\". This captures the same core textual change with no discrepancy in the original or corrected text. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:26:55.462679",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:03:23.043004",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 33,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The predicted edit does the same core punctuation insertion but includes additional surrounding context (\"at length said\" \u2192 \"at length, said\"). This preserves the essential intended correction (adding the comma after \"length\" before \"said\").\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "with some hurried words to the effect that she thanked him from her heart",
"corrected_text": "indeed thanked him from her heart",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"That's well! I hopedso; I am not disappointed.\"",
"corrected_text": "\"I hoped so; I am not disappointed.\"",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "as she did",
"corrected_text": "as she did,",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hopedso",
"corrected_text": "I hoped so",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:19:22.580510",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:47:08.791404",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 33,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"length said\") and the same corrected text (\"length, said\"), capturing the identical core change: insertion of a comma after \"length\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 36,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"I hope so\" with \"I hoped so\", capturing the same core change (tense shift from present to past). Line number differences are to be ignored, and no other text discrepancies exist."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "with some hurried words to the effect that she thanked him from her heart",
"corrected_text": "indeed thanked him from her heart",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "as she",
"corrected_text": "as she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:20:00.780947",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:05:32.097776",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 11,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same transformation: \"heart so\" \u2192 \"heart, so\" (inserting a comma after \"heart\"). This captures the core intended change with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 33,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth inserts a comma after \"length\" in the phrase \"length said\" \u2192 \"length, said\". The prediction applies the same core punctuation change\u2014adding a comma after \"length\"\u2014but includes one extra preceding word (\"at\") as context: \"at length said\" \u2192 \"at length, said\". This preserves the essential intended correction (comma insertion after \"length\" before \"said\")."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 36,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n\nText Content Accuracy: Both edits replace the exact same original text \"I hope so\" with the same corrected text \"I hoped so.\" This captures the core intended change (tense shift from present to past). Line numbers are to be ignored, so the difference in line_number does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "thanked him from her heart (as she did, if it be the source of earnestness)",
"corrected_text": "indeed thanked him from her heart (as she did, if it be the source of earnestness)",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:27:25.423492",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:03:58.866299",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: adding a comma after \"hers\" in the phrase \"hers and\" to make it \"hers, and\". The core punctuation correction is identical. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers\". The original_text and corrected_text match the ground truth\u2019s core change exactly.\n\nLine numbers are ignored as instructed, so they do not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 26,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits apply the same core change to the same text span: \"moment with\" is corrected to \"moment, with\" by inserting a comma after \"moment\". The essential punctuation change is identical. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this, he thought directly, \"they",
"corrected_text": "At this, he thought directly, \"They",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:20:31.049030",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:47:44.937754",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change: \"hers and\" \u2192 \"hers, and\" (inserting a comma after \"hers\"). The essential edit is identical. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"hers matters\" to \"hers, matters\", inserting a comma after \"hers\". The core punctuation correction is identical. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this, he thought directly, \"they",
"corrected_text": "At this, he thought directly, \"They",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:21:00.101292",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:06:24.906157",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"hers and\" to \"hers, and\", inserting a comma after \"hers\". This captures the same core punctuation correction as the ground truth. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core transformation: inserting a comma after \"hers\" in the phrase \"hers matters\" to make it \"hers, matters\". The original_text and corrected_text match the ground truth\u2019s intended change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this, he thought directly, \"they",
"corrected_text": "At this, he thought directly, \"They",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:27:51.402363",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:04:35.966309",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in our fancy",
"corrected_text": "that, in our fancy,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "is now.\"",
"corrected_text": "is now.\" \"Thank you,\"",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:21:28.917005",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:48:16.717925",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in our fancy",
"corrected_text": "that, in our fancy,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "is now.\"",
"corrected_text": "is now.\" \"Thank you,\"",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:21:56.263005",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:07:06.229921",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in our fancy",
"corrected_text": "that, in our fancy,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "is now.\"",
"corrected_text": "is now.\" \"Thank you,\"",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:28:36.426741",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:05:14.595682",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same core change: adding a period to the end of \"Chapter XXIX\" to make it \"Chapter XXIX.\". The original and corrected texts match the ground truth exactly, so the intended edit is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits indicate the same core change: \"city\" \u2192 \"City\". There is no discrepancy in the original or corrected text, and the essential capitalization correction is captured precisely.\n\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 11,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change, replacing \"them,\" with \"them;\". The essential edit is identical. Line number differences are ignored per instructions."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 17,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both edits change \"existence,\" to \"existence;\". This captures the same core punctuation correction (comma to semicolon) with identical original and corrected text.\nTherefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless,",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:22:43.899807",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:48:56.564431",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same core change: adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\". The original and corrected texts match the ground truth intention precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"city\" is corrected to \"City.\" There is no discrepancy in the original or corrected text. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, replacing \"them,\" with \"them;\". The essential punctuation correction is identical. Line numbers are ignored as instructed, so the mismatch there does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth edit type.\n\nText Content Accuracy: Both ground truth and prediction change the same token from \"existence,\" to \"existence;\". The core punctuation change (comma to semicolon) is identical, with matching original and corrected text. Therefore, the predicted edit correctly captures the intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless,",
"corrected_text": "motionless;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,",
"corrected_text": "view;",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:23:28.392849",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:07:49.704156",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a period to \"Chapter XXIX\" to form \"Chapter XXIX.\". The original and corrected texts are identical between ground truth and prediction, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same core change: \"city\" is corrected to \"City\". There is no discrepancy in the original or corrected text content.\n\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change from \"them,\" to \"them;\". Since the essential edit is identical, the prediction matches the ground truth intention. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core correction: changing \"existence,\" to \"existence;\". The original and corrected text strings are identical between ground truth and prediction, capturing the intended punctuation change precisely."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless,",
"corrected_text": "motionless,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,",
"corrected_text": "view,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:29:11.688414",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:05:50.746301",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest snorts\" to \"driest of snorts\" (i.e., corrects the spelling from \"dryest\" to \"driest\" and inserts \"of\" before \"snorts\"). The predicted edit performs the same core transformation: \"dryest snorts\" \u0000 \"driest of snorts\". Although the ground truth includes surrounding context (\"of the\" and the final period) and the prediction omits that extra context, the essential intended change is captured correctly."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "\"Your own recompense for all this, Mr. Pancks,\" said Clennam, \"ought to be a large one.\"",
"corrected_text": "\"Your own recompense for all this, Mr. Pancks,\" said Clennam, \"ought to be a large one.\"",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:24:04.316724",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:49:32.431388",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest snorts\" to \"driest of snorts\" (i.e., corrects spelling dryest\u2192driest and inserts \"of\" before \"snorts\"). The prediction makes the same core change: \"dryest snorts\" \u2192 \"driest of snorts\". Although the ground truth includes additional surrounding context (\"of the\" and the ending period), the prediction still captures the essential intended correction without introducing a different change.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "\"Your own recompense for all this, Mr. Pancks,\" said Clennam, \"ought to be a large one.\"",
"corrected_text": "\"Your own recompense for all this, Mr. Pancks,\" said Clennam, \"ought to be a large one.\"",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:24:34.142922",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:08:33.186951",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts \"of\" so that \"dryest snorts\" becomes \"driest of snorts\" (within the larger phrase \"of the ... .\"). The prediction captures the same core transformation: \"dryest snorts\" \u2192 \"driest of snorts\". Although it omits surrounding context (\"of the\" and the final period), it correctly represents the essential correction intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "of",
"corrected_text": "of",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "\"Your own recompense for all this, Mr. Pancks,\" said Clennam, \"ought to be a large one.\"",
"corrected_text": "\"Your own recompense for all this, Mr. Pancks,\" said Clennam, \"ought to be a large one.\"",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T10:29:39.198263",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:06:28.612831",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction likewise inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the core intended punctuation change in the same location. Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". The essential punctuation correction is identical. Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:25:02.038672",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T10:50:08.562454",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction likewise inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the core intended punctuation change in the correct location."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same transformation: \"quickly that\" \u2192 \"quickly, that\", inserting a comma after \"quickly\". The core change is identical and fully captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-sonnet-4.6",
"date": "2026-03-08T11:25:31.262735",
"annotator": "Claude Sonnet 4.6",
"annotation_date": "2026-03-08T11:09:14.223280",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"For Mr. Pancks\" to \"For, Mr. Pancks\", i.e., it inserts a comma after \"For\". The predicted edit changes \"For Mr.\" to \"For, Mr.\", which also inserts a comma after \"For\" in the same location. Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the core intended punctuation change accurately."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". This captures the core intended correction with identical original and corrected text. Line numbers are irrelevant per instructions."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bedkroom",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2026-02-17",
"release_source": "https://www.anthropic.com/news/claude-sonnet-4-6",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "Claude Sonnet 5",
"model_id": "claude-sonnet-5",
"date": "2026-07-23T15:19:02.327795",
"shots": 2,
"config": {
"model_id": "claude-sonnet-5",
"display_name": "Claude Sonnet 5",
"shots": 2,
"temperature": 0.0,
"date": "2026-07-23",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:14.497995",
"file_results": [
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:37.713414",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:14.497995",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The core intended punctuation change is captured identically. Therefore, the predicted edit matches the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both edits change \"yonder to-day\" to \"yonder, to-day\" by inserting a comma after \"yonder\". The core punctuation change is identical in the prediction and matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change, inserting a comma after \"pockets\" (\"pockets and\" \u2192 \"pockets, and\"). The core punctuation correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:36.417827",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:27:48.381858",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. This captures the identical core punctuation change with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the core intended change of inserting a comma after \"howling\". Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original phrase \"yonder to-day\" and the same correction \"yonder, to-day\", inserting a comma after \"yonder\". This captures the identical core change intended by the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"Sir\" to \"sir,\" capturing the same core capitalization correction with identical original and corrected text. No mismatch or missing element is present."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to make it \"pockets, and.\" The original_text and corrected_text match the ground truth\u2019s core change exactly. Therefore the predicted edit correctly captures the intended correction."
}
],
"file_id": "003"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:39.454160",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:28:24.236872",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a period to \"Fellow Travellers\" to make it \"Fellow Travellers.\". The original and corrected texts match the ground truth intention precisely. Therefore, the prediction is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change of inserting a comma after \"howling\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the intended punctuation change of inserting a comma after \"yonder\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to make it \"pockets, and\". The original and corrected texts match exactly, so the intended punctuation correction is captured correctly."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:30.789148",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:15:58.381670",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same change from \"No sir\" to \"No, sir\", inserting a comma after \"No\". The core intended punctuation correction is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:33.879915",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:27:38.463529",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"No sir\" to \"No, sir\" (inserting a comma after \"No\"). The core change is identical, with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"why did he dine\") and the same corrected text (\"Why did he dine\"), capturing the core intended change of capitalizing the initial 'w' to 'W'. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!\"",
"corrected_text": "cold stones?\"",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the punctuation from an exclamation mark to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction captures the same core change (\"!\" \u2192 \"?\") and simply includes an additional trailing quotation mark in both the original and corrected strings (\"cold stones!\\\"\" \u2192 \"cold stones?\\\"\"). This added context does not alter the essential correction and still reflects the same intended punctuation change.\n\nTherefore, the prediction correctly captures the ground truth edit."
}
],
"file_id": "004"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:36.737232",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:28:16.932174",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"No sir\" to \"No, sir\", inserting a comma after \"No\". This captures the core intended correction exactly. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 25,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The predicted type is \"capitalization,\" which exactly matches the ground truth.\n\n2) Text Content Accuracy: The ground truth edit capitalizes the initial word in the phrase, changing \"why did he dine\" to \"Why did he dine.\" The prediction captures the core intended change by converting \"why\" to \"Why.\" Although it provides less surrounding context, it correctly identifies and applies the essential capitalization change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is converting an exclamation mark to a question mark at the end of the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction captures the same core punctuation change at the same word-ending: \"stones!\" \u2192 \"stones?\". Although the prediction omits the preceding word \"cold\", it still correctly represents the essential intended correction (replace the terminal \"!\" with \"?\")."
}
],
"file_id": "004"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:30.425716",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:06.114440",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the edit type matches exactly. 2) Text Content Accuracy: Both specify the identical transformation \"if he\" \u2192 \"If he,\" capturing the same core capitalization change with no discrepancies. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the core intended change by replacing \"said\" \u2192 \"asked\". Although it omits the unchanged surrounding context (\"Clennam\"), it still represents the essential correction accurately."
}
],
"file_id": "005"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:33.401452",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:27:41.092236",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"if he\" to \"If he\", capturing the identical core capitalization correction. No discrepancies in the original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both identify the same original text (\"said Clennam\") and the same corrected text (\"asked Clennam\"). The core change\u2014replacing \"said\" with \"asked\" in the phrase\u2014is captured identically. Therefore, the predicted edit matches the ground truth intention."
}
],
"file_id": "005"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:37.723582",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:28:17.175026",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": 33,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"if he\" to \"If he\", capturing the identical core capitalization correction. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"said Clennam\") and the same corrected text (\"asked Clennam\"), capturing the identical core change. Line numbers are ignored as instructed, so the mismatch there does not affect correctness."
}
],
"file_id": "005"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:26.208379",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:12.818099",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road,\" capturing the identical core change (inserting a hyphen between the words). No mismatch in original or corrected text content."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction applies the same core change\u2014adding a comma immediately after \"said\"\u2014but includes extra preceding context (\"and said\" \u2192 \"and said,\"). This preserves the essential intended correction (comma after \"said\"), so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials,costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about,and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "tion,did",
"corrected_text": "tion did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly,telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said,when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam,in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:25.967089",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:27:49.362311",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"high road\") and the same corrected text (\"high-road\"), capturing the identical core change of hyphenating the phrase. Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"said\" to \"said,\", adding a comma after the word. The core change is identical with no discrepancies. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials,costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about,and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "tion,did",
"corrected_text": "tion did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly,telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said,when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam,in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:16.910263",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:28:22.870938",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road,\" capturing the identical core punctuation/hyphenation change. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 4,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change of inserting a comma after \"materials\". Therefore, the prediction matches the ground truth\u2019s intended edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma to change \"about and\" to \"about, and\". The original and corrected text strings match the ground truth intention exactly. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change\u2014adding a comma after \"earnestly\". Line numbers are ignored as instructed. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 21,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"said\" in the phrase \"said when\", yielding \"said, when\". The original_text and corrected_text match the ground truth\u2019s intended punctuation change. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "said \"Clennam!",
"corrected_text": "said, \"Clennam!",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction adds the same comma in the same position, merely including additional surrounding context (\"said \\\"Clennam!\" \u2192 \"said, \\\"Clennam!\"). This captures the core intended punctuation change."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The prediction also inserts a comma after \"Clennam\" while including extra surrounding context (\"Clennam in amazement\" \u2192 \"Clennam, in amazement\"). This captures the same core punctuation correction intended by the ground truth.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion did",
"corrected_text": "tion, did",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:58.681070",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:15:55.190935",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period: \"Sun and Shadow\" \u2192 \"Sun and Shadow.\" The predicted edit makes the same core change with identical original and corrected text. The additional note about the handwritten annotation does not alter the edit content. Therefore, the prediction captures the essential intended correction."
}
],
"file_id": "007"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:02.651826",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:27:29.462984",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period after \"Shadow\". The prediction captures the same original text and the same corrected text with the period added. The note about a handwritten query does not alter the core change.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
}
],
"file_id": "007"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:16:57.858990",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:28:16.159207",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 2,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so this matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period: \"Sun and Shadow\" \u2192 \"Sun and Shadow.\" The prediction specifies the same original and corrected text, capturing the identical core punctuation change. The extra note about a handwritten annotation does not alter the essential edit.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
}
],
"file_id": "007"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:17.314307",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:15.777366",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation: \"Let Loose\" \u2192 \"Let Loose.\" (adding a period). The core intended change is captured identically. Therefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original segment (\"dull autumn\") and the same correction (\"dull, autumn\"), inserting a comma after \"dull\". The core change is captured identically.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 8,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"man slowly\") and the same corrected text (\"man, slowly\"), capturing the identical core change (inserting a comma after \"man\"). Line number differences are ignored as instructed. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 8,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original phrase \"Chalons was\" and the same corrected phrase \"Chalons, was\", capturing the core intended change (inserting a comma after \"Chalons\"). Ignoring line numbers, the predicted edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:13.090361",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:27:38.113161",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The core change is identical with no discrepancies. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"dull autumn\") and the same corrected text (\"dull, autumn\"), capturing the identical core change of inserting a comma after \"dull\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 12,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"man slowly\" to \"man, slowly\" by inserting a comma after \"man\". This captures the same core correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 12,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Chalons was\" to \"Chalons, was\", inserting a comma after \"Chalons\". The core textual change is identical. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:20.286168",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:28:28.681072",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"Let Loose\" to \"Let Loose.\" (adding a period). This captures the core intended correction with no discrepancy in the text content. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core punctuation change: inserting a comma after \"man\" (\"man slowly\" \u2192 \"man, slowly\"). The essential edit is identical. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"Chalons was\" \u2192 \"Chalons, was\", inserting a comma after \"Chalons\". This captures the core intended change with identical original and corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden with wet",
"corrected_text": "soddened with wet",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"sodden\" to \"soddened\". The prediction changes \"sodden with wet\" to \"soddened with wet\", which includes additional surrounding context but preserves the same core substitution (\"sodden\" \u2192 \"soddened\"). Therefore, the essential intended correction is correctly captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:33.773783",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:35.319343",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so much",
"corrected_text": "heart, so much",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"heart\" in the sequence \"heart so\" \u2192 \"heart, so.\" The prediction applies the same core punctuation change (inserting a comma after \"heart\") but includes additional surrounding context: \"heart so much\" \u2192 \"heart, so much.\" This preserves the essential intended edit, merely extending the span to include the following word.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"length said\" to \"length, said\", inserting a comma after \"length\". The core punctuation correction is identical. Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change (present tense to past tense). Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart (as she did",
"corrected_text": "thanked him from her heart indeed (as she did",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:30.298187",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:28:17.121765",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so much",
"corrected_text": "heart, so much",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth correction inserts a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The prediction applies the same core change\u2014adding a comma after \"heart\"\u2014but includes extra surrounding context (\"much\"). This still captures the essential intended edit (inserting the comma after \"heart\")."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said Clennam",
"corrected_text": "at length, said Clennam",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction performs the same core change, adding a comma after \"length\" before \"said,\" but includes additional surrounding context (\"at\" before and \"Clennam\" after). This still captures the essential intended correction.\n\nTherefore, the predicted edit correctly matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change from present to past tense. No discrepancies in the essential edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart (as she did",
"corrected_text": "thanked him from her heart indeed (as she did",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:47.803654",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:28:41.987973",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so much",
"corrected_text": "heart, so much",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The prediction also inserts a comma after \"heart\" but includes extra surrounding context (\"heart so much\" \u2192 \"heart, so much\"). This preserves the same core punctuation correction at the same location, so it captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The core punctuation correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope so",
"corrected_text": "hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"I hope so\" \u2192 \"I hoped so\", i.e., the core correction is changing the verb form \"hope\" to \"hoped\" while keeping \"so\". The prediction captures the same essential change: \"hope so\" \u2192 \"hoped so\". Although it omits the leading \"I\" context, it still accurately represents the intended correction (hope \u2192 hoped) and does not introduce a different meaning or a different edit.\n\nTherefore, the predicted edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart (as she did",
"corrected_text": "thanked him from her heart, indeed, (as she did",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:43.864614",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:50.773946",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"hers and\" to \"hers, and\" by inserting a comma after \"hers\". The core punctuation change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change: insertion of a comma after \"hers\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"moment with\" to \"moment, with,\" inserting a comma after \"moment.\" The core change is identical with no discrepancies."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "idea. At this,",
"corrected_text": "idea.\n\nAt this,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this",
"corrected_text": "At this",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:56.649267",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:28:30.138407",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and thanked",
"corrected_text": "hers, and thanked",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" (\"hers and\" \u2192 \"hers, and\"). The predicted edit also inserts a comma after \"hers\" (\"hers and thanked\" \u2192 \"hers, and thanked\"). Although the prediction includes additional surrounding context (\"thanked\"), it captures the same core punctuation change at the same location. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters little",
"corrected_text": "hers, matters little",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The predicted edit also inserts a comma after \"hers\" (\"hers matters little\" \u2192 \"hers, matters little\"), adding extra surrounding context (\"little\") but preserving the same essential punctuation change at the same location. Therefore, it captures the core intention of the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with a look",
"corrected_text": "moment, with a look",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"moment\" (\"moment with\" \u2192 \"moment, with\"). The predicted edit also inserts a comma after \"moment\" (\"moment with a look\" \u2192 \"moment, with a look\"). Although the prediction includes additional surrounding words (\"a look\"), it preserves the same core punctuation change at the same location. \u0628\u0646\u0627\u0628\u0631\u0627\u06cc\u0646 the essential edit is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that it was",
"corrected_text": "beautiful, that it was",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"beautiful\" (\"beautiful that\" \u2192 \"beautiful, that\"). The prediction applies the same core punctuation change\u2014adding a comma immediately after \"beautiful\"\u2014but includes additional surrounding words (\"beautiful that it was\" \u2192 \"beautiful, that it was\"). This added context does not alter the essential edit, so it correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they are to be married",
"corrected_text": "They are to be married",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the word \"they\" to \"They\". The predicted edit applies the same capitalization change but includes additional surrounding context (\"they are to be married\" \u2192 \"They are to be married\"). The core intended change\u2014capitalizing the initial \"they\"\u2014is correctly captured."
}
],
"file_id": "010"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:50.889428",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:29:06.808740",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction change \"hers and\" to \"hers, and\" by inserting a comma after \"hers.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the core intended change\u2014adding a comma after \"hers\". Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth edit type.\n\nText Content Accuracy: Both ground truth and prediction change \"moment with\" to \"moment, with\" by inserting a comma after \"moment.\" The original and corrected text strings match exactly, capturing the core intended punctuation change."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same corrected text (\"beautiful, that\"), capturing the core intended change\u2014adding a comma after \"beautiful\". Therefore, the prediction fully matches the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they are to be married",
"corrected_text": "They are to be married",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates capitalizing the word \"they\" to \"They\". The prediction applies the same capitalization change to the same word but includes additional surrounding context (\"they are to be married\" \u2192 \"They are to be married\"). This preserves the core intended change (lowercase to uppercase initial \"they\").\n\nTherefore, the predicted edit correctly captures the ground truth\u2019s intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "At this, he thought directly",
"corrected_text": "\n\nAt this, he thought directly",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "010"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:56.673608",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:56.586812",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same change from \"shore and\" to \"shore, and,\" inserting a comma after \"shore.\" The essential punctuation correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once were",
"corrected_text": "than once were",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\" edits, so the type matches.\n2) Text Content Accuracy: The ground truth intends to change the word \"that\" to \"than\". The prediction changes \"that once were\" to \"than once were\", which preserves the same core substitution (that \u2192 than) while including extra surrounding context. This correctly captures the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in our fancy",
"corrected_text": "that, in our fancy,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:57.118564",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:28:31.361258",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cThank you!\u201d murmured Clennam",
"corrected_text": "\u201cThank you,\u201d murmured Clennam",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation,\" and the predicted edit type is also \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The core change in the ground truth is replacing the exclamation mark in \"Thank you!\" with a comma (\"Thank you,\"). The prediction applies the same punctuation change within a longer quoted phrase (\"Thank you!\" \u2192 \"Thank you,\") while including additional surrounding context (quotation marks and \"murmured Clennam\"). This extra context does not alter the essential correction, and the intended punctuation change is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the identical core change: inserting a comma after \"shore\" in the phrase \"shore and\" to make it \"shore, and\". The original_text and corrected_text match exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "things that once",
"corrected_text": "things than once",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction are of type \"replacement\", so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth intends to replace the word \"that\" with \"than\". The predicted edit replaces the phrase \"things that once\" with \"things than once\", which includes the same core substitution (\"that\" \u2192 \"than\") while adding extra surrounding context. This still captures the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:17:49.821194",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:29:05.384510",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you!\" murmured Clennam",
"corrected_text": "\"Thank you,\" murmured Clennam",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "things that once were",
"corrected_text": "things than once were",
"observed_line_number": 47,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:18:48.669148",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:58.656554",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, converting \"city\" to \"City.\" There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of the same",
"corrected_text": "return to the same",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the word \"of\" to \"to\". The predicted edit changes the phrase \"return of the same\" to \"return to the same\", which includes additional surrounding context but preserves the same core substitution (\"of\" \u2192 \"to\"). Therefore, the prediction captures the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit applies the same core change\u2014adding a comma immediately after \"existence\"\u2014while including extra surrounding context (\"existence at\" \u2192 \"existence, at\"). This captures the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction adds that same comma in context (\"motionless when\" \u2192 \"motionless, when\"). Although the prediction includes an extra following word for context, it captures the same core punctuation change at the same location. \u05dc\u05db\u05df, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma after \"view\" within a larger span (\"view by\" \u2192 \"view, by\"). This preserves the same core punctuation change (inserting a comma immediately after \"view\") while including additional surrounding context, which is acceptable.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation immediately following the word \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The predicted edit applies a semicolon right after \"existence\" as well (\"existence is\" \u2192 \"existence; is\"), adding surrounding context (the following word \"is\"). Although the prediction does not explicitly show the comma in its original span, it captures the core intended change: a semicolon should appear after \"existence\" at that location. Therefore the essential punctuation correction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:19:02.327795",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:28:33.035603",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both indicate the same core change: \"city\" is corrected to \"City\". The original and corrected texts match the ground truth\u2019s intended capitalization change with no discrepancy.\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of",
"corrected_text": "return to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth changes \"of\" to \"to\". The prediction changes \"return of\" to \"return to\", which includes additional surrounding context but preserves the same core substitution (\"of\" \u2192 \"to\"). Therefore, it captures the intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change, adding a comma immediately after \"existence\" within a slightly larger context (\"existence at\" \u2192 \"existence, at\"). This captures the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth adds a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit adds the same comma after \"motionless\" while including extra following context (\"motionless when\" \u2192 \"motionless, when\"). This captures the same core punctuation change intended by the ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change, adding a comma immediately after \"view\" within a longer phrase (\"view by\" \u2192 \"view, by\"). This preserves the essential intended punctuation correction, merely with additional surrounding context.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:18:54.568566",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:29:13.518628",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly. Text Content Accuracy: Both specify the same core change, converting \"city\" to \"City\". There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of",
"corrected_text": "return to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\" edits, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces \"of\" with \"to\". The predicted edit replaces \"return of\" with \"return to\", which includes extra surrounding context but preserves the same core change (\"of\" \u2192 \"to\"). Therefore, the prediction accurately captures the intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence",
"corrected_text": "busy existence,",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit adds a comma after the phrase \"busy existence\" (\"busy existence\" \u2192 \"busy existence,\") which still places the comma immediately after \"existence\". This includes extra surrounding context but captures the same essential punctuation change."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core correction: adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same core change\u2014adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). There is no discrepancy in the text content or intended correction. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates changing the punctuation following the word \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction gives \"existence\" \u2192 \"existence;\". Although the prediction omits the comma in the original_text field, it still captures the core intended change: the punctuation after \"existence\" should be a semicolon. The corrected outcome matches the ground truth's intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:18:28.050810",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:17:10.002291",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly. Text Content Accuracy: Both target the same text \"him\" with no change in wording, indicating the intent is to apply italics to \"him\". The predicted edit captures the same core change (formatting) as the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "score; now I have paid it.",
"corrected_text": "score; now, I have paid it.",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"now\" in the phrase \"now I\" \u2192 \"now, I\". The predicted edit applies the same core punctuation change within a larger surrounding snippet: \"score; now I have paid it.\" \u2192 \"score; now, I have paid it.\" This preserves the essential intended correction (adding the comma after \"now\").\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both identify the same original text (\"authorize\") and the same corrected text (\"authorise\"), capturing the identical core spelling change from American to British form. No discrepancy in the essential edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "boiling-over",
"corrected_text": "boiling over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at twenty:",
"corrected_text": "at twenty;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now to break",
"corrected_text": "you, now to break",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:18:30.703686",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:28:50.553957",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"replacement\" and the predicted edit type is also \"replacement\", so the type matches.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and adds \"of\" before \"snorts\" (\"dryest snorts\" \u2192 \"driest of snorts\"), within the larger phrase \"of the ... snorts.\" The prediction targets the key substring \"dryest snorts\" and replaces it with \"driest of snorts\", which captures the same essential spelling correction and insertion. Although the prediction omits surrounding context (\"of the\") and the trailing period, it still represents the core intended change accurately."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\nText Content Accuracy: Both identify the same target text \"him\" with no textual change other than applying italics. The core intended change (italicizing the word \"him\") is captured correctly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "score; now I have paid it.",
"corrected_text": "score; now, I have paid it.",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after \"now\" in the phrase \"now I\" \u2192 \"now, I\". The prediction makes the same core punctuation change, adding a comma after \"now\" within a longer surrounding context (\"score; now I have paid it.\" \u2192 \"score; now, I have paid it.\"). Including additional context is acceptable, and the essential edit is identical."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as a \"replacement\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"authorize\") and the same corrected text (\"authorise\"), capturing the identical core change (American to British spelling). Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty:",
"corrected_text": "twenty;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:18:32.594397",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:29:25.046174",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "*him*",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"italicize\", which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth indicates that the word \"him\" should be italicized (with no change to the literal characters). The prediction also targets \"him\" and represents the italicization explicitly as \"*him*\". This captures the same core intended change (apply italics) without altering the underlying word. Therefore, the essential edit is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I have paid",
"corrected_text": "now, I have paid",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The predicted edit also inserts a comma after \"now\" and simply includes extra surrounding context (\"now I have paid\" \u2192 \"now, I have paid\"). The core punctuation change is identical and correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both change the same original text \"authorize\" to the same corrected text \"authorise\", capturing the identical core correction. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "little",
"corrected_text": "*little*",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty,",
"corrected_text": "twenty;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:18:19.775194",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T11:16:55.284321",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is the insertion of a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The predicted edit likewise inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the core intended punctuation change in the correct location. Therefore, the essential edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 9,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"quickly that\") and the same correction (\"quickly, that\"), capturing the identical core change\u2014adding a comma after \"quickly\". Therefore the predicted edit matches the ground truth\u2019s intention. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:18:19.999821",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T12:28:22.900646",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same corrected text (\"For, Mr. Pancks\"), capturing the identical core change: insertion of a comma after \"For\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 9,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"quickly that\") and the same corrected text (\"quickly, that\"), capturing the identical core change: insertion of a comma after \"quickly\". Since line numbers are to be ignored, the prediction fully matches the ground truth intent."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me; not",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "claude-sonnet-5",
"date": "2026-07-23T15:18:23.712534",
"annotator": "Claude Sonnet 5",
"annotation_date": "2026-07-23T13:28:55.272579",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same corrected text (\"For, Mr. Pancks\"), capturing the identical core change\u2014adding a comma after \"For\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Anthropic",
"release_date": "2026-06-30",
"release_notes": "Claude Sonnet 5 shipped as Anthropic's new default model at the end of June 2026; the OpenRouter listing appeared 2026-06-30 and press coverage ran 2026-07-01.",
"display_suffix": "*",
"display_note": "Claude results use resized/recompressed images when needed to stay below Anthropic's 5 MiB per-image limit."
},
{
"model_name": "DeepSeek R1",
"model_id": "or_deepseek_r1",
"date": "2026-02-22T15:42:07.961833",
"shots": 2,
"config": {
"model_id": "or_deepseek_r1",
"display_name": "DeepSeek R1",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-07",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:05:35.722930",
"file_results": [
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:40:39.512877",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:05:35.722930",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dark cold",
"corrected_text": "dark, cold",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mediterranean",
"corrected_text": "Mediterranean",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea mist",
"corrected_text": "sea-mist",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "however that",
"corrected_text": "however, that",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "stonework",
"corrected_text": "stone-work",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "roman",
"corrected_text": "Roman",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "window which",
"corrected_text": "window, which",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:40:40.603546",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:24:54.939714",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "darkness and",
"corrected_text": "darkness, and",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english channel",
"corrected_text": "English Channel",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "flower pots",
"corrected_text": "flower-pots",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ship wreck",
"corrected_text": "ship-wreck",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mediterranean",
"corrected_text": "Mediterranean",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:40:43.338008",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:25:31.133419",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:40:44.341347",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:07:20.621208",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sunlight",
"corrected_text": "sun-light",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english",
"corrected_text": "English",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "coast",
"corrected_text": "Coast",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bread and",
"corrected_text": "bread, and",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "winebarrels",
"corrected_text": "wine-barrels",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:40:44.635252",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:25:42.529965",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:40:44.636185",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:26:24.555432",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moon light",
"corrected_text": "moon-light",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mediterranean",
"corrected_text": "Mediterranean",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ship however",
"corrected_text": "ship, however",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "forecastle",
"corrected_text": "fo'c'sle",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "waves and",
"corrected_text": "waves, and",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:40:57.332226",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:09:44.512640",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "OF",
"corrected_text": "OF",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ONE CHAPTER",
"corrected_text": "ONE, CHAPTER",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "1",
"corrected_text": "1.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "MARSEILLES THE",
"corrected_text": "MARSEILLES\u2014THE",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "spoke and",
"corrected_text": "spoke, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Vecchia so",
"corrected_text": "Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:40:57.331202",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:27:21.139274",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sunlight and",
"corrected_text": "sunlight, and",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "doorway where",
"corrected_text": "doorway, where",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "well built",
"corrected_text": "well-built",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english channel",
"corrected_text": "English Channel",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiled and",
"corrected_text": "smiled, and",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:08.026030",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:28:33.506440",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:09.283808",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:10:13.992710",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "rain fall",
"corrected_text": "rain-fall",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english channel",
"corrected_text": "English Channel",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tree; and",
"corrected_text": "tree, and",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "slowly",
"corrected_text": "quickly",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "bookstore",
"corrected_text": "book store",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:10.992603",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:31:00.515079",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dark blue",
"corrected_text": "dark-blue",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english Channel",
"corrected_text": "English Channel",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "however when",
"corrected_text": "however, when",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "santos dumont",
"corrected_text": "Santos Dumont",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Santos Dumont",
"corrected_text": "Santos-Dumont",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "head ache",
"corrected_text": "headache",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "its",
"corrected_text": "it's",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:02.155455",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:29:38.296786",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fed looking",
"corrected_text": "fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:02.153206",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:11:21.463767",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sail boat",
"corrected_text": "sail-boat",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english channel",
"corrected_text": "English Channel",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and then",
"corrected_text": "and, then",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "tomatoe",
"corrected_text": "tomato",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "captain smith",
"corrected_text": "Captain Smith",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "fastly",
"corrected_text": "quickly",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:12.516118",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:31:45.941707",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:14.715595",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:33:23.257439",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sun light",
"corrected_text": "sun-light",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "atlantic ocean",
"corrected_text": "Atlantic Ocean",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "however he",
"corrected_text": "however, he",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:16.748526",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:12:08.285533",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sun light",
"corrected_text": "sun-light",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "paris",
"corrected_text": "Paris",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "then he",
"corrected_text": "then, he",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:16.749860",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:34:08.952612",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:21.727429",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:34:29.481825",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sunlight and",
"corrected_text": "sunlight, and",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "quickly",
"corrected_text": "swiftly",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "well known",
"corrected_text": "well-known",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mediterranean",
"corrected_text": "Mediterranean",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea side",
"corrected_text": "sea-side",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "in to",
"corrected_text": "into",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:21.726120",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:13:05.231527",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:33.553208",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:35:16.533888",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sun light",
"corrected_text": "sun-light",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dark,blue",
"corrected_text": "dark blue",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "captain smith",
"corrected_text": "Captain Smith",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea;and",
"corrected_text": "sea, and",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "broken--",
"corrected_text": "broken\u2014",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:33.551920",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:36:48.066733",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "door way",
"corrected_text": "door-way",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "mr",
"corrected_text": "Mr.",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "paris",
"corrected_text": "Paris",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "head quarters",
"corrected_text": "head-quarters",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "francis",
"corrected_text": "Francis",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty four",
"corrected_text": "twenty-four",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:37.766698",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:14:55.118620",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:34.953536",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:38:09.425837",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Tale of Two Cities",
"corrected_text": "A Tale of Two Cities",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sunlight and",
"corrected_text": "sunlight, and",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "head quarters",
"corrected_text": "head-quarters",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "doorway where",
"corrected_text": "doorway, where",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english",
"corrected_text": "English",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "the prisoners",
"corrected_text": "captives",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty five",
"corrected_text": "twenty-five",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "after noon",
"corrected_text": "afternoon",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:35.520446",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:40:01.461872",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sunset and",
"corrected_text": "sunset, and",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "well known",
"corrected_text": "well-known",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english channel",
"corrected_text": "English Channel",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "however when",
"corrected_text": "however, when",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "postscript ps",
"corrected_text": "Postscript PS",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "head ache",
"corrected_text": "headache",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "e mail",
"corrected_text": "email",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "world war ii",
"corrected_text": "World War II",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mr. darcy",
"corrected_text": "Mr. Darcy",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:40.785739",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:19:57.328990",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "however when",
"corrected_text": "however, when",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "united states",
"corrected_text": "United States",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "brother in law",
"corrected_text": "brother-in-law",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mr. brown",
"corrected_text": "Mr. Brown",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tea time",
"corrected_text": "tea-time",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "finished and",
"corrected_text": "finished, and",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english breakfast",
"corrected_text": "English breakfast",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:40.981935",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:40:05.227492",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moon light",
"corrected_text": "moon-light",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english channel",
"corrected_text": "English Channel",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ships that",
"corrected_text": "ships, that",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "harbor",
"corrected_text": "harbour",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "waves and",
"corrected_text": "waves, and",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:40.785252",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:44:54.900447",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sun light",
"corrected_text": "sun-light",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "however when",
"corrected_text": "however, when",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english channel",
"corrected_text": "English Channel",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "well known",
"corrected_text": "well-known",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "paris france",
"corrected_text": "Paris France",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Paris France",
"corrected_text": "Paris, France",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:42.655980",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:22:19.494409",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "however when",
"corrected_text": "however, when",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english",
"corrected_text": "English",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "television",
"corrected_text": "TV",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:54.021419",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:45:40.440425",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "chapter three",
"corrected_text": "CHAPTER THREE",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "river bank",
"corrected_text": "river-bank",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "its",
"corrected_text": "it's",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english coast",
"corrected_text": "English Coast",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "well known",
"corrected_text": "well-known",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tomorrow morning",
"corrected_text": "tomorrow, morning",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:54.025167",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:23:13.572763",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "darkness fell",
"corrected_text": "darkness, fell",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mediterranean",
"corrected_text": "Mediterranean",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea breeze",
"corrected_text": "sea-breeze",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "harbor",
"corrected_text": "harbour",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sails rigging",
"corrected_text": "sails, rigging",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "La Jeune Am\u00e9lie",
"corrected_text": "_La Jeune Am\u00e9lie_",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:58.448984",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:41:43.814853",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "postoffice",
"corrected_text": "post-office",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english",
"corrected_text": "English",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said to",
"corrected_text": "said, to",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Paris but",
"corrected_text": "Paris, but",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mediterranean",
"corrected_text": "Mediterranean",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "captain",
"corrected_text": "Captain",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea port",
"corrected_text": "sea-port",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "deck however",
"corrected_text": "deck, however",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:42:03.154367",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:47:04.906609",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "gatsby",
"corrected_text": "Gatsby",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shes",
"corrected_text": "she's",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "twenty first",
"corrected_text": "twenty-first",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "DArcy",
"corrected_text": "Darcy",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tree and",
"corrected_text": "tree, and",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english",
"corrected_text": "English",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "sun-set",
"corrected_text": "sunset",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:42:05.108064",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:24:07.011374",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "head and",
"corrected_text": "head, and",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine shop",
"corrected_text": "wine-shop",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "head down",
"corrected_text": "head, down",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "monseigneur",
"corrected_text": "Monseigneur",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "monseigneur",
"corrected_text": "Monseigneur",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "good graces",
"corrected_text": "good-graces",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:42:07.961833",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T19:44:07.602217",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "in paris",
"corrected_text": "in Paris",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cell,which",
"corrected_text": "cell, which",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "postoffice",
"corrected_text": "post-office",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shes",
"corrected_text": "she's",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english channel",
"corrected_text": "English Channel",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "captain nemo",
"corrected_text": "Captain Nemo",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_deepseek_r1",
"date": "2026-02-22T15:41:58.451049",
"annotator": "DeepSeek R1",
"annotation_date": "2025-04-07T20:47:48.752330",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "doorway and",
"corrected_text": "doorway, and",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "postoffice",
"corrected_text": "post-office",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english channel",
"corrected_text": "English Channel",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "well known",
"corrected_text": "well-known",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "afterwards he",
"corrected_text": "afterwards, he",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "DeepSeek",
"release_date": "2025-01-20",
"release_source": "https://api-docs.deepseek.com/news/news250120"
},
{
"model_name": "DeepSeek V3 0324",
"model_id": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:41.681387",
"shots": 2,
"config": {
"model_id": "or_deepseek_chat_v3_0324",
"display_name": "DeepSeek V3 0324",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:18:06.299635",
"file_results": [
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:18.329286",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:18:06.299635",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting to be fed looking",
"corrected_text": "was waiting to be fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:18.329992",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:24:44.243290",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:22.296878",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:18:34.036599",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting to be fed looking",
"corrected_text": "was waiting to be fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:24.569863",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:19:10.446663",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting to be fed looking",
"corrected_text": "was waiting to be fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:29.003850",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:34:49.097499",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:29.302550",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T05:56:39.665128",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:32.400341",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:19:41.166582",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:34.644666",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:36:07.218724",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said he",
"corrected_text": "said, he",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english",
"corrected_text": "English",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said I",
"corrected_text": "said, I",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said he",
"corrected_text": "said, he",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said I",
"corrected_text": "said, I",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said he",
"corrected_text": "said, he",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said I",
"corrected_text": "said, I",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said he",
"corrected_text": "said, he",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said I",
"corrected_text": "said, I",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said he",
"corrected_text": "said, he",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:39.100224",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T05:57:06.599523",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said he",
"corrected_text": "said, he",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english",
"corrected_text": "English",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "very",
"corrected_text": "extremely",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "then and",
"corrected_text": "then, and",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "important",
"corrected_text": "_important_",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "however he",
"corrected_text": "however, he",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "quite",
"corrected_text": "",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:36.491768",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:21:01.786024",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "south east",
"corrected_text": "south-east",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "south west",
"corrected_text": "south-west",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mediterranean",
"corrected_text": "Mediterranean",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea and",
"corrected_text": "sea, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea which",
"corrected_text": "sea, which",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea and",
"corrected_text": "sea, and",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea and",
"corrected_text": "sea, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea and",
"corrected_text": "sea, and",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea and",
"corrected_text": "sea, and",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sea and",
"corrected_text": "sea, and",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:50.796291",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T05:58:25.340465",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "traveller's",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "travellers'",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "travellers",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "travellers",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "travellers",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "travellers",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "travellers",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "travellers",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "travellers",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "travelers",
"corrected_text": "travellers",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:51.109792",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:21:48.415639",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting to be fed looking",
"corrected_text": "was waiting to be fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:53.408848",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T05:58:53.036242",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:39:53.407265",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:38:25.901834",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:03.266715",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T05:59:39.461606",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting to be fed looking",
"corrected_text": "was waiting to be fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:03.619362",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:23:37.016653",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting to be fed looking",
"corrected_text": "was waiting to be fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:09.427094",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:38:57.971571",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sun rose",
"corrected_text": "sun, rose",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mediterranean",
"corrected_text": "Mediterranean",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "blue water",
"corrected_text": "azure sea",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "deck and",
"corrected_text": "deck, and",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Santa Maria",
"corrected_text": "Santa Maria",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "the port",
"corrected_text": "the busy port",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "very",
"corrected_text": "",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:06.397363",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T06:00:36.770997",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:13.959632",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:24:55.018336",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said I",
"corrected_text": "said, I",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "english",
"corrected_text": "English",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man and",
"corrected_text": "man, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "prisoner said",
"corrected_text": "prisoner, said",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Yes I",
"corrected_text": "Yes, I",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "italian",
"corrected_text": "Italian",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No I",
"corrected_text": "No, I",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "years I",
"corrected_text": "years, I",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "prisoner said",
"corrected_text": "prisoner, said",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "a",
"corrected_text": "an",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:14.873343",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:39:45.347454",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting",
"corrected_text": "was waiting,",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:13.452685",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T06:00:54.997006",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:14.514089",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:25:21.372429",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:11.706929",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:40:41.713994",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting to be fed looking",
"corrected_text": "was waiting to be fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:25.379244",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:25:42.441238",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:25.380601",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:41:33.234947",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting to be fed looking",
"corrected_text": "was waiting to be fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:41.681387",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T06:02:19.700099",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sun rose",
"corrected_text": "sun rose,",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "mediterranean",
"corrected_text": "Mediterranean",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "blue sea",
"corrected_text": "azure sea",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "waves which",
"corrected_text": "waves, which",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "the horizon",
"corrected_text": "the distant horizon",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sail appeared",
"corrected_text": "sail appeared.",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "very",
"corrected_text": "",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:39.284468",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:26:29.522931",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_deepseek_chat_v3_0324",
"date": "2026-02-22T15:40:39.282836",
"annotator": "DeepSeek V3 0324",
"annotation_date": "2025-04-08T00:42:34.349457",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "church bells",
"corrected_text": "church bells,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "wine bottles",
"corrected_text": "wine-bottles",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "got through",
"corrected_text": "got, through",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "iron bars fashioned",
"corrected_text": "iron bars, fashioned",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "grating where",
"corrected_text": "grating, where",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "outside and",
"corrected_text": "outside; and",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "intact in",
"corrected_text": "intact, in",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "indian ocean",
"corrected_text": "Indian Ocean",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "was waiting to be fed looking",
"corrected_text": "was waiting to be fed; looking",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bars that",
"corrected_text": "bars, that",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "marseilles harbour",
"corrected_text": "Marseilles Harbour",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 13,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 14,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Civita Vecchia so",
"corrected_text": "Civita Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "DeepSeek",
"release_date": "2025-03-25",
"release_source": "https://api-docs.deepseek.com/news/news250325"
},
{
"model_name": "DeepSeek V4 Flash Vision Exp",
"model_id": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:22.623544",
"shots": 2,
"config": {
"model_id": "or_deepseek_v4_flash_vision_exp",
"display_name": "DeepSeek V4 Flash Vision Exp",
"shots": 2,
"temperature": 0.0,
"reasoning_effort": "high",
"date": "2026-09-06",
"notes": "Benchmark run with 2-shot learning at high reasoning effort; incomplete at 35/36 evaluation predictions after repeated provider timeouts"
},
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:17:12.767227",
"file_results": [
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:08.863854",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:17:12.767227",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "N",
"corrected_text": "\"N",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:01.989870",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:29:34.704876",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "O more",
"corrected_text": "NO more",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:09.349443",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T02:18:54.347893",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "O",
"corrected_text": "NO",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "TRAVELLERS",
"corrected_text": "TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:16.399935",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:17:16.277288",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:05.453082",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:30:23.198536",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day,",
"corrected_text": "to-day?",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:10.317992",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T02:18:15.779079",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones.\"",
"corrected_text": "stones?\"",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:12.179432",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:17:41.950694",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the verb \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam.\" The predicted edit changes \"said\" \u2192 \"asked\" without including the following context \"Clennam.\" This still captures the core intended change (replacing \"said\" with \"asked\"). The omission of surrounding context does not alter the essential correction being made.\n\nTherefore, the predicted edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "he",
"corrected_text": "If he",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:08.995174",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:44:20.722656",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the verb in the phrase \"said Clennam\" to \"asked Clennam\". The prediction changes \"said\" to \"asked\" without including the following context (\"Clennam\"), but it captures the essential intended change (said \u2192 asked). The omitted context does not alter the core correction, and \"Clennam\" would remain unchanged.\n\nTherefore, the predicted edit correctly captures the ground truth\u2019s intention."
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:08.914222",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:57:52.097855",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the type matches.\n2) Text Content Accuracy: The ground truth changes the verb phrase from \"said Clennam\" to \"asked Clennam\". The predicted edit changes \"said\" to \"asked\" without including the following word \"Clennam\" in the span. This still captures the core intended correction (replacing \"said\" with \"asked\"); the omitted context does not alter the essential edit. Therefore the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "he",
"corrected_text": "If he",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:07.149289",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:28:00.341913",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam!",
"corrected_text": "Clennam!,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "mean?",
"corrected_text": "mean?,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "broad",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "and",
"corrected_text": "",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:19.270791",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:31:41.854640",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction applies the same core change\u2014adding a comma immediately after \"said\"\u2014but includes additional preceding context (\"and said\" \u2192 \"and said,\"). This preserves the essential intended edit (comma insertion after \"said\").\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "amazement.",
"corrected_text": "amazement,",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:09.015951",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:42:56.272853",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clemnam",
"corrected_text": "Clemnam,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "highroad",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:32.052511",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:17:33.878045",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW",
"corrected_text": "SUN AND SHADOW.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:32.460644",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:30:19.358728",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW.",
"corrected_text": "SUN AND SHADOW",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:32.464738",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:43:30.156396",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW",
"corrected_text": "SUN AND SHADOW.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:40.943253",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:18:15.176785",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "wet,",
"corrected_text": "wet, lythen",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "sodden lythen",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:32.054300",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:40:44.228605",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late",
"corrected_text": "A late",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Chalons",
"corrected_text": "Chalons a",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar-trees",
"corrected_text": "poplar-trees broken",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:45.878899",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:47:26.492082",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "he",
"corrected_text": "he",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "with",
"corrected_text": "with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "the",
"corrected_text": "the",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "was",
"corrected_text": "was",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:41.268761",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:18:08.284638",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hurried words",
"corrected_text": "hurried indeed words",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "And, now, Minnie Gowan,",
"corrected_text": "I hope so And, now, Minnie Gowan,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:46.915887",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:30:36.792289",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "rais-",
"corrected_text": "rais- indeed",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"will",
"corrected_text": "\"will I hope so",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:47.700741",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:43:16.089341",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "she thanked",
"corrected_text": "she indeed thanked",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:57.556221",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:17:38.233923",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and",
"corrected_text": "and,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:57.527128",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:58:08.528421",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation, inserting a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The core change is identical, with no discrepancy in original or corrected text."
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:07.153897",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:18:03.020812",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "fancy, whatever",
"corrected_text": "fancy, in whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"Thank you,\" murmured",
"corrected_text": "Thank you, \"Thank you,\" murmured",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "withindoors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:03.213227",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:30:20.308386",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "is now.",
"corrected_text": "is now.\"Thank you\"",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "that my",
"corrected_text": "that in my",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within",
"corrected_text": "within-doors",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:03.872548",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:44:04.312684",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "in our whatever",
"corrected_text": "in our fancy whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "now.\" \"Thank you,\"",
"corrected_text": "now.\" \"Thank you,\" \"Thank you,\"",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:00.631481",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:17:56.746114",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "THE",
"corrected_text": "The",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "lucant return",
"corrected_text": "lucant to return",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:49:55.922636",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:30:15.543799",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "a stand-still",
"corrected_text": "a to stand-still",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:09.541854",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T02:17:49.243104",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "The house",
"corrected_text": "The cof house",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "piece of clockwork.",
"corrected_text": "to piece of clockwork.",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:07.449718",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:20:45.594513",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW",
"corrected_text": "SUN AND SHADOW.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "and",
"corrected_text": "",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:20.494497",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:30:22.125090",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 36,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"authorize\" with \"authorise\". This captures the same core spelling change intended in the ground truth. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of the dryest snorls",
"corrected_text": "driest of snorls",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:17.212775",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:44:12.128607",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the type as \"italicize\", so the edit type matches exactly.\nText Content Accuracy: Both specify italicizing the same text, \"him\" \u2192 \"him\" (format change only). The core intended change\u2014applying italics to the word \"him\"\u2014is captured identically. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"authorize\") and the same corrected text (\"authorise\"), capturing the identical core spelling change. No mismatch in the intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:16.829130",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:28:24.156169",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Panks",
"corrected_text": "Pancks",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:16.387245",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:30:40.526928",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_deepseek_v4_flash_vision_exp",
"date": "2026-09-06T02:50:22.623544",
"annotator": "DeepSeek V4 Flash Vision Exp",
"annotation_date": "2026-09-06T01:45:47.187655",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "bedroom",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "another back",
"corrected_text": "another back",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Good gracious, Arthur",
"corrected_text": "Good gracious, Arthur",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a faded check",
"corrected_text": "a faded check",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "making me that",
"corrected_text": "making me not that",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "is making me",
"corrected_text": "is making me",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "In times for ever fled Arthur,",
"corrected_text": "In times for ever fled Arthur,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at least",
"corrected_text": "at least,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "never be recalled",
"corrected_text": "never be recalled",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "bed",
"corrected_text": "bed",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "For",
"corrected_text": "For",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "DeepSeek",
"release_date": "2026-08-21",
"release_source": "https://api-docs.deepseek.com/news/news260821/",
"release_notes": "Experimental vision-enabled DeepSeek V4 Flash model. Benchmark configuration uses high reasoning effort.",
"display_suffix": "*",
"display_note": "Incomplete run: 35 of 36 evaluation predictions completed; one request repeatedly timed out at the provider."
},
{
"model_name": "GLM 4.6V",
"model_id": "or_glm_4_6v",
"date": "2026-02-22T14:16:09.096409",
"shots": 2,
"config": {
"model_id": "or_glm_4_6v",
"display_name": "GLM 4.6V",
"shots": 2,
"temperature": 0.0,
"date": "2026-02-22",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:04:54.918185",
"file_results": [
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:37.951108",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:04:54.918185",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "n",
"corrected_text": "N",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:25.941708",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:07:04.968741",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:26.282595",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:09:00.620898",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:25.941610",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:09:07.721021",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:26.737776",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:09:00.644677",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:30.541724",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:04:41.163091",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The evaluation involves two criteria: edit type accuracy and text content accuracy. \n\n1. Edit Type Accuracy: The ground truth and the predicted edit both specify 'capitalization' as the type of edit. This matches exactly, fulfilling the requirement for edit type accuracy.\n\n2. Text Content Accuracy: The ground truth edit changes the text from 'why did he dine' to 'Why did he dine', specifically involving the capitalization of the first word from 'why' to 'Why'. The predicted edit focuses on the core change by highlighting the adjustment of 'why' to 'Why', which is the essential modification indicated by the ground truth. The prediction captures the essential capitalization change despite providing less context than the ground truth, which is permissible as per the guidelines.\n\nSince both the edit type and the essential text content change are accurately represented in the prediction, the predicted edit correctly captures the intention of the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit based on both criteria. Firstly, the edit type is accurate, as both the ground truth and the prediction indicate a 'punctuation' type of edit. Secondly, the text content accuracy is also correct, as both the ground truth and the prediction reflect the change from 'cold stones!' to 'cold stones?'. This matches precisely, capturing the core change of replacing the exclamation mark with a question mark. Thus, both criteria are satisfied, and the prediction can be considered correct."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:26.344040",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:09:00.636187",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:28.784853",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:04:59.533724",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The edit type in both the ground truth and the predicted edit is 'replacement', which matches exactly, satisfying the first criterion for edit type accuracy. For text content accuracy, the core change identified by both the ground truth and the prediction is replacing 'said' with 'asked'. While the ground truth includes the additional context 'Clennam', this does not alter the fundamental transformation being applied to the text. Both scenarios capture the essential edit of changing \u2018said\u2019 to \u2018asked,\u2019 which means the prediction correctly interprets the main intention of the ground truth edit. As both criteria are met, the predicted edit is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "If",
"corrected_text": "I can",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:26.282482",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:04:20.586233",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "If",
"corrected_text": "I cap",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:43.134764",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:08:03.745277",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:54.480072",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:05:46.647416",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The edit type in both the ground truth and the predicted edit is 'punctuation'. Since the types match exactly, the prediction meets the edit type criterion. 2. Text Content Accuracy: The core change in the ground truth is the insertion of a comma after 'said', changing 'said when' to 'said, when'. The predicted edit captures this essential change by showing a transition from 'said' to 'said,' which includes the insertion of the comma after 'said'. Although the predicted edit doesn't mention 'when', it is still considered accurate according to the text content criterion since it captures the core change required, which is the addition of the punctuation after 'said'. As both criteria are satisfactorily met, the prediction is deemed correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "alive!",
"corrected_text": "alive!,",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "highway",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "her",
"corrected_text": "",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:50.093886",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:06:23.849755",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "her",
"corrected_text": "",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:53.786577",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:10:27.914879",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:47.489701",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:06:48.076295",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit and the ground truth edit both accurately identify the type of edit as 'punctuation', focusing on the addition of a period to the phrase 'Sun and Shadow'. Both edits correctly change 'Sun and Shadow' to 'Sun and Shadow.', capturing the core modification needed. The line number is intentionally ignored for this evaluation, and hence does not affect the correctness of the prediction. The purpose of the edit\u2014adding punctuation at the end of the sentence\u2014is maintained. Therefore, both the edit type and the text content match the ground truth's intention accurately."
}
],
"file_id": "007"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:14:56.592191",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:07:03.952342",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit and the ground truth both have the same edit type, which is 'punctuation'. This matches accurately with the requirement for edit type accuracy. For text content accuracy, the ground truth shows a change from 'Sun and Shadow' to 'Sun and Shadow.', indicating an addition of a period at the end. The predicted edit also reflects this change from 'Sun and Shadow' to 'Sun and Shadow.', capturing the core correction of adding a period. Therefore, the prediction accurately captures the essential change as shown in the ground truth, despite any potential differences in line numbers, which are ignored in this evaluation. Both criteria for correctness\u2014edit type and text content\u2014are satisfied, leading to the conclusion that the prediction is correct."
}
],
"file_id": "007"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:00.753138",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:08:22.390522",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late, dull autumn night",
"corrected_text": "A late, dull autumn night",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:00.436839",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:10:54.991451",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:09.893902",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:09:00.678627",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:17.411558",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:09:11.245612",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "generous",
"corrected_text": "",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hoped so",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:15.179829",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:10:03.560421",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "generous",
"corrected_text": "",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart",
"corrected_text": "thanked him indeed from her heart",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:23.026885",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:10:13.996865",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "she thanked him from her heart",
"corrected_text": "indeed she thanked him from her heart",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "generous",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:21.717564",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:10:25.649320",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters little",
"corrected_text": "hers, matters little",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with a look",
"corrected_text": "moment, with a look",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "idea.",
"corrected_text": "idea,",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:22.283489",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:10:32.732649",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "from hers and",
"corrected_text": "from hers, and",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters little",
"corrected_text": "on hers, matters little",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "for a moment with a look",
"corrected_text": "for a moment, with a look",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sorrow for him she was so beautiful",
"corrected_text": "sorrow for him, she was so beautiful",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "peace\u2014or ill for his peace",
"corrected_text": "peace\u2014or ill, for his peace",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:27.345485",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:10:42.705894",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers, and",
"corrected_text": "hers and",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers, matters",
"corrected_text": "hers matters",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment, with",
"corrected_text": "moment with",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sorrow for him, she",
"corrected_text": "sorrow for him she",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:39.602771",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:10:51.883047",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 25,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "The predicted edit captures the essential change specified in the ground truth. Both edits are of type 'replacement', which is an exact match, satisfying the first criterion of edit type accuracy. For the text content accuracy, both the ground truth and the prediction performed the same core change, replacing 'within doors' with 'within-doors'. The predicted edit correctly interprets the necessary text modification, accomplishing the key aspect of replacing a space with a hyphen as intended by the ground truth. Therefore, both criteria are fully met, making the prediction correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 30,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "The evaluation criteria require us to judge the prediction based on two factors: edit type accuracy and text content accuracy. \n\n1. **Edit Type Accuracy:** The predicted edit type is 'replacement,' which matches the ground truth's edit type of 'replacement.' Therefore, the edit type accuracy is fulfilled.\n\n2. **Text Content Accuracy:** The prediction specifies a change from 'that' to 'than,' which matches the core change described in the ground truth. This indicates that the key modification in text content, which is replacing 'that' with 'than,' is accurately captured by the prediction. Additional context or line numbers are irrelevant for this evaluation as stipulated by the instructions.\n\nSince both the edit type and the textual content modification align with the ground truth, the prediction is deemed correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "whatever",
"corrected_text": "",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "Thank you,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:40.067223",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:10:53.931784",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 27,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit and the ground truth edit match in both the edit type and the text content. \n\n1. Edit Type Accuracy: The type 'replacement' is consistent in both the ground truth and the predicted edit, indicating the prediction successfully identified the type of correction required.\n\n2. Text Content Accuracy: The core change involves replacing 'within doors' with 'within-doors'. Both the ground truth and the predicted edit capture this exact change, demonstrating an accurate recognition of the intended modification. Additional context, and the discrepancy in line numbers are intentionally ignored as per the evaluation instructions, and do not affect the accuracy of the content change itself.\n\nTherefore, taking into consideration all these aspects, the prediction captures both the type and content of the correction accurately."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 32,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit is correct in terms of edit type and text content accuracy. \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edit have the same 'type': 'replacement'. This matches exactly, fulfilling the requirement for edit type accuracy.\n\n2. Text Content Accuracy: The ground truth specifies a replacement of the word 'that' with 'than'. The prediction matches this exactly, as it also indicates the replacement of 'that' with 'than'. The essential change is correctly captured in the predicted edit since it changes the same word in the same way. \n\nSince both criteria \u2013 edit type and text content accuracy \u2013 are met, the prediction correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "whatever",
"corrected_text": "",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:47.707356",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:11:33.694173",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 32,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit successfully matches the ground truth edit in both criteria that are relevant for this evaluation. Firstly, the edit type is 'replacement' in both the ground truth and the predicted edit, satisfying the requirement for Edit Type Accuracy. Secondly, regarding Text Content Accuracy, the core change involves replacing 'that' with 'than'. The predicted edit captures this essential change correctly, even if additional context like surrounding text is considered, meaning the primary alteration (changing 'that' to 'than') was accurately identified. Therefore, the prediction aligns with the intentions of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "in our fancy whatever Pet was",
"corrected_text": "",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:48.334015",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:11:26.017612",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 2,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "The predicted edit matches the ground truth edit in terms of type and text content. First, the edit type for both the ground truth and the prediction is 'replacement', which indicates they are of the same type. Second, the original text 'of' and the corrected text 'to' are identical in both cases. Therefore, the core change that the ground truth identifies is accurately captured by the predicted edit, as it reflects the same transformation from 'of' to 'to'. Since both criteria (edit type and text content) are satisfied, the prediction is deemed correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "liege",
"corrected_text": "",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:50.574678",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:12:38.610548",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "liege",
"corrected_text": "",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "xxix",
"corrected_text": "XXIX",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "to",
"corrected_text": "",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:56.682701",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:11:33.042018",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit correctly matches the ground truth in terms of edit type and text content accuracy. Both edits are classified as 'capitalization' edits, which matches the requirement for edit type accuracy. For text content accuracy, the essential change is from 'city' to 'City', which is precise and consistent in both the ground truth and the predicted edit. Therefore, despite a discrepancy in line numbers, which is not considered in this evaluation, the predicted edit accurately captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "busy existence",
"corrected_text": "busy existence,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "liege",
"corrected_text": "",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:49.021202",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:11:38.371446",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 4,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "The predicted edit and the ground truth edit both identify the change type as 'replacement', which matches exactly and thereby satisfies the criterion for Edit Type Accuracy. For Text Content Accuracy, the core change identified by the ground truth is the transformation from 'of the dryest snorts.' to 'of the driest of snorts.', focusing specifically on replacing 'dryest' with 'driest of'. The predicted edit captures this essential change by transforming 'dryest snorts' into 'driest of snorts', which includes the key modification 'dryest' \u2192 'driest of'. Even though the predicted edit does not include 'of the' at the beginning and 'of' at the end, this additional context is not necessary for capturing the core change. Therefore, since both criteria are satisfied, the prediction correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:51.601597",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:12:02.274179",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 4,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the predicted edit have the type 'replacement'. This means the edit type matches exactly, fulfilling the first criterion.\n\n2. Text Content Accuracy: The core change being captured involves replacing 'dryest snorts' with 'driest of snorts.' The predicted edit has successfully identified this change, despite capturing only 'dryest snorts' instead of 'of the dryest snorts' from the ground truth. However, since the essential change 'dryest' to 'driest of' is accurately reflected, and additional context (words before/after the change) is permissible, the prediction meets the text content requirement.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit, fulfilling both criteria."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:15:58.047036",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:12:00.231038",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 5,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The predicted edit type is 'replacement', which matches the ground truth's edit type of 'replacement'. Thus, the edit type is correct. \n\n2. Text Content Accuracy: The core change in both the ground truth and the prediction involves altering 'dryest snorts' to 'driest of snorts'. While the ground truth includes an article 'of the dryest snorts' compared to the prediction 'dryest snorts', the essential modification from 'dryest' to 'driest of' is maintained in the prediction. The presence of additional context ('of the' versus none) in the ground truth does not negate that the primary correction focus\u2014the phrase 'dryest'\u2014is accurately transformed in both cases. Therefore, the content of the edit is correct. \n\nSince both the edit type and the text content are accurately captured in the prediction, the overall edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:16:00.811382",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:12:25.550484",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "making me,",
"corrected_text": "making me",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cucumber",
"corrected_text": "Cucumber",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:16:00.909728",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:14:07.360518",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_glm_4_6v",
"date": "2026-02-22T14:16:09.096409",
"annotator": "GLM 4.6V",
"annotation_date": "2026-02-22T14:12:25.617393",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bed room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me,",
"corrected_text": "making me",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "Zhipu AI",
"release_date": "2025-12-08"
},
{
"model_name": "GPT-4 Turbo",
"model_id": "gpt-4-turbo",
"date": "2026-02-22T15:21:47.776515",
"shots": 2,
"config": {
"model_id": "gpt-4-turbo",
"display_name": "GPT-4 Turbo",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:09:09.857564",
"file_results": [
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:07.383538",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:09:09.857564",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "over yonder, to-day, Sir, is there?",
"corrected_text": "howling over yonder, to-day, Sir, is there?",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:02.281713",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:11:48.518589",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder, to-day, Sir, is there",
"corrected_text": "yonder, to-day, Sir, is there?",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:01.470921",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:15:23.605995",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder, to-day, Sir, is there",
"corrected_text": "yonder, to-day, Sir, is there?",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:08.204693",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:10:01.292285",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "misgiving",
"corrected_text": "reason",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:15.886105",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:12:49.645279",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine to",
"corrected_text": "why did he deign to",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "CAP",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "remark in margin",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:07.552054",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:16:27.171055",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine to",
"corrected_text": "why did he deign to",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "remark in margin",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:01.619134",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:10:52.644999",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:15.961174",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:13:47.402159",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 34,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"if he\" to \"If he,\" which is specifically capitalizing the word \"if\" at the start of the phrase. The prediction changes \"if\" to \"If,\" capturing the same core capitalization change, just with less surrounding context. Since the essential correction (lowercase \"if\" \u2192 capitalized \"If\") is correctly represented, the text content matches the ground truth's intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"said Clennam\" to \"asked Clennam\", whose core correction is replacing the word \"said\" with \"asked\" while leaving \"Clennam\" unchanged. The prediction replaces \"said\" with \"asked\" (without including the following context \"Clennam\"), which still captures the essential intended change. Including less surrounding text is acceptable as long as the core replacement is correct.\n\nTherefore, the predicted edit correctly reflects the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:01.469291",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:17:29.461838",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:23.727429",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:11:55.108018",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "found, and on his",
"corrected_text": "found, and on his",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:25.724661",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:14:41.805033",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both edits change \"said\" to \"said,\" by adding a comma. This captures the core intended correction with identical original and corrected text. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:33.873274",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:18:29.337393",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after \"said\" in the phrase \"said when\" \u2192 \"said, when\". The predicted edit \"said\" \u2192 \"said,\" captures the same core punctuation insertion (adding a comma immediately after \"said\"). Although it omits the following word \"when\" from the edit span, it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:30.181341",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:12:47.820490",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the same core punctuation correction with identical original and corrected text. Line number differences are ignored as instructed."
}
],
"file_id": "007"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:29.913253",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:15:36.416022",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the same core punctuation correction. Line numbers are ignored as instructed."
}
],
"file_id": "007"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:30.020852",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:19:20.540147",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the same core punctuation correction. Line numbers are ignored as instructed."
}
],
"file_id": "007"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:39.741553",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:13:51.223112",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "gloomy place, reflected",
"corrected_text": "gloomy place, reflected",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone it was wet",
"corrected_text": "Saone it was wet,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with wet",
"corrected_text": "sodden with wet;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:40.764152",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:16:33.091994",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "looking-glass in",
"corrected_text": "looking-glass, in",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "river Saone it",
"corrected_text": "river Saone, it",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with",
"corrected_text": "sodden, with",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:42.384426",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:20:26.179994",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "looking-glass in",
"corrected_text": "looking-glass, in",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "river Saone",
"corrected_text": "river Sa\u00f4ne",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with",
"corrected_text": "sodden, with",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:52.433211",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:14:57.039117",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I would have",
"corrected_text": "I wouldhave",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hoped",
"corrected_text": "hope",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hope so",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:54.705542",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:17:37.180057",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I would have",
"corrected_text": "I wouldhave",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "generous",
"corrected_text": "",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hoped",
"corrected_text": "I hoped",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hope so",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:52.480657",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:21:29.109067",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I would have",
"corrected_text": "I wouldhave",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hope so",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:20:50.168525",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:15:57.705769",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they are to be married.",
"corrected_text": "They are to be married.",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hers",
"corrected_text": "cap",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:01.017345",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:18:30.986941",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hers",
"corrected_text": "his",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:05.251621",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:22:22.098857",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hers",
"corrected_text": "his",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:18.985246",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:16:47.104511",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "that in",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "that once",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:12.933954",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:19:35.378892",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "that once were",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:16.649851",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:23:21.246599",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "that once were",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:32.840000",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:17:55.500184",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city (CCCP)",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "lucrative",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:20.644389",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:20:45.498627",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 19,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit inserts a comma in the same location within a larger span of text (\"view by\" \u2192 \"view, by\"). This includes extra context but preserves the core intended change: adding a comma immediately after \"view.\" \u0907\u0938\u0932\u093f\u090f the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city (CCCP)",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "lucrative",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence; at",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:20.949167",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:24:22.438285",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city (CCCP)",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "lucrative",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:19.034237",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:18:55.318775",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "driest of snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "boiling-over",
"corrected_text": "boiling over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:32.004865",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:21:45.734499",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "bubbling out of him",
"corrected_text": "bubbling out of him",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "driest of snorts",
"corrected_text": "driest of snorts.",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:34.514685",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:25:23.213441",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 28,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction identify the same target text \"him\" and the corrected text remains \"him\" with the intention to apply italics. This captures the same core change (formatting \"him\" in italics). Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "driest of snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "boiling-over",
"corrected_text": "boiling over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:41.922103",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-07T21:19:56.165108",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 9,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". This captures the core intended change precisely. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sight as this and",
"corrected_text": "sight as this, and",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me",
"corrected_text": "making me,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "apology",
"corrected_text": "apology,",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur",
"corrected_text": "Arthur,",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a",
"corrected_text": "",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:42.344489",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T00:22:57.607840",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 9,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"quickly that\" to \"quickly, that\" by inserting a comma after \"quickly.\" This captures the same core punctuation correction as the ground truth. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sight as this and",
"corrected_text": "sight as this, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me",
"corrected_text": "making me,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "apology",
"corrected_text": "apology,",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "excuse it when",
"corrected_text": "excuse it, when",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur",
"corrected_text": "Arthur,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "though unquestionably distant still",
"corrected_text": "though unquestionably distant, still",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and if I did I suppose it would depend considerably on the nature of the view but",
"corrected_text": "and if I did, I suppose it would depend considerably on the nature of the view, but",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a",
"corrected_text": "",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-4-turbo",
"date": "2026-02-22T15:21:47.776515",
"annotator": "GPT-4 Turbo",
"annotation_date": "2025-04-08T07:26:30.733904",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 9,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same transformation: \"quickly that\" \u2192 \"quickly, that\", i.e., inserting a comma after \"quickly\". This captures the core intended edit with identical original and corrected text.\n\nLine numbers are ignored per instructions, so the discrepancy there does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sight as this and",
"corrected_text": "sight as this, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me",
"corrected_text": "making me,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "apology",
"corrected_text": "apology,",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "excuse it when",
"corrected_text": "excuse it, when",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur",
"corrected_text": "Arthur,",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "though unquestionably distant still",
"corrected_text": "though unquestionably distant, still",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view but",
"corrected_text": "view, but",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a",
"corrected_text": "",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2023-11-06",
"release_source": "https://openai.com/index/new-models-and-developer-products-announced-at-devday/"
},
{
"model_name": "GPT-4.1",
"model_id": "gpt_4_1",
"date": "2026-02-22T15:30:19.242396",
"shots": 2,
"config": {
"model_id": "gpt_4_1",
"display_name": "GPT-4.1",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-14",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:01:46.386970",
"file_results": [
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:22.428148",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:01:46.386970",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over yonder to-day, sir; is there?",
"corrected_text": "howling, over yonder, to-day, sir; is there?",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth core change is inserting a comma between \"howling\" and \"over\" (\"howling over\" \u2192 \"howling, over\"). The prediction includes a larger span of surrounding text, but it does make the same essential change: \"howling over ...\" \u2192 \"howling, over ...\". Although the prediction also introduces additional commas elsewhere (e.g., after \"yonder\" and \"to-day\"), it still captures the ground truth\u2019s intended punctuation change at the target location.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "pockets and rattling",
"corrected_text": "pockets, and rattling",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"pockets\" in the phrase \"pockets and\" \u2192 \"pockets, and\". The prediction applies the same core punctuation change in a longer span: \"pockets and rattling\" \u2192 \"pockets, and rattling\". This includes extra surrounding context but preserves the essential intended edit (adding a comma after \"pockets\").\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:39.750903",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:16:03.736710",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "pockets and rattling",
"corrected_text": "pockets, and rattling",
"observed_line_number": 14,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth correction inserts a comma after the word \"pockets\" (\"pockets and\" \u2192 \"pockets, and\"). The predicted edit makes the same essential change, inserting a comma after \"pockets\" (\"pockets and rattling\" \u2192 \"pockets, and rattling\"). The prediction includes additional surrounding context (\"rattling\"), but the core punctuation change is identical to the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers .",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over yonder to-day, sir; is there?",
"corrected_text": "howling, over yonder, to-day, sir; is there?",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:30.896000",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:53:45.819507",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "pockets and rattling",
"corrected_text": "pockets, and rattling",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"pockets\" in the phrase \"pockets and\" \u2192 \"pockets, and.\" The prediction makes the same core punctuation change\u2014adding a comma after \"pockets\"\u2014but includes additional surrounding context (\"rattling\"). Since the essential edit (\"pockets\" \u2192 \"pockets,\") is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over yonder to-day, sir; is",
"corrected_text": "howling over yonder to-day, sir, is",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:30.074935",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:02:42.064439",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "why did he dine to-",
"corrected_text": "Why did he dine to-",
"observed_line_number": 27,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial word \"why\" to \"Why\" in the phrase \"why did he dine\". The predicted edit applies the same capitalization change (\"why\" \u2192 \"Why\") but includes additional trailing context (\"to-\"). Including extra surrounding words is acceptable as long as the core change is the same, which it is here.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:28.837310",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:16:58.528140",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "why did he dine to-",
"corrected_text": "Why did he dine to-",
"observed_line_number": 27,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial word \"why\" to \"Why\" in the phrase \"why did he dine\". The prediction applies the same capitalization change (\"why\" \u2192 \"Why\") but includes additional trailing context (\"to-\"). This added context does not change the core intended correction; it still captures the essential capitalization edit correctly.\n\nTherefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:29.283626",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:54:36.742326",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why did he dine to-",
"corrected_text": "Why did he dine to-",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the initial \"why\" to \"Why\" in the phrase \"why did he dine\". The prediction makes the same core capitalization change (\"why\" \u2192 \"Why\") and simply includes additional trailing context (\"to-\") on both the original and corrected sides. Since the essential intended change\u2014capitalizing the first word\u2014is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:31.489465",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:03:32.966167",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 34,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase \"if he\" \u2192 \"If he\". The prediction captures the same core change by capitalizing \"if\" \u2192 \"If\". Although it omits the following word (\"he\"), it still represents the essential intended correction: changing the lowercase \"if\" to uppercase \"If\" at that location. Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 37,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the core intended change by replacing \"said\" with \"asked\" (even though it omits the trailing context \"Clennam\"). Since the essential correction is the same word substitution, the text content matches the ground truth's intention."
}
],
"file_id": "005"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:32.211168",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:17:41.697214",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"said Clennam\" to \"asked Clennam\", whose core correction is replacing the word \"said\" with \"asked\". The predicted edit replaces \"said\" with \"asked\" (without including the trailing context \"Clennam\"). Omitting surrounding context is acceptable as long as the essential change is captured, and it is. Therefore, the prediction matches the ground truth\u2019s intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:48.053065",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:55:28.171885",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 33,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"capitalization,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to capitalize the word \"if\" at the beginning of the phrase \"if he\" \u2192 \"If he.\" The prediction captures the same core change by converting \"if\" \u2192 \"If.\" Although it includes less surrounding context (omitting \"he\"), the essential intended correction\u2014capitalizing \"if\"\u2014is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the verb phrase from \"said Clennam\" to \"asked Clennam,\" whose core intended correction is replacing \"said\" with \"asked\" while keeping the rest of the phrase the same. The predicted edit replaces \"said\" with \"asked\" without including the trailing context (\"Clennam\"), but it still captures the essential change identified in the ground truth. Including less surrounding text is acceptable as long as the core change is correct."
}
],
"file_id": "005"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:58.306262",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:04:32.916397",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 10,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: \"about and\" \u2192 \"about, and\" (inserting a comma after \"about\"). This captures the core intended correction with identical original and corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The corrected text matches the ground truth\u2019s intended punctuation change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 32,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both edits change the same original text \"Clennam in\" to the same corrected text \"Clennam, in,\" inserting a comma after \"Clennam.\" This captures the core intended correction exactly. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:00.810579",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:18:36.248948",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 11,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same correction (\"about, and\"), capturing the identical core change\u2014adding a comma after \"about\". Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"said\" is corrected to \"said,\" by adding a comma. The essential edit is identical. Line numbers are ignored as instructed, so the mismatch there does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 32,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction make the same core change: inserting a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The original and corrected text strings match exactly, so the intended punctuation correction is correctly captured.\n\nLine numbers are ignored as instructed, so the differing line_number values do not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:28:56.646582",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:56:19.218337",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"about and\" to \"about, and\", inserting a comma after \"about\". The core punctuation correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:05.845097",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:05:27.768308",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:05.189973",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:19:22.151832",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:06.111963",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:57:17.347463",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:07.799792",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:06:20.044209",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "wet,",
"corrected_text": "wet,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late, dull autumn night",
"corrected_text": "A late, dull autumn night",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:21.560449",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:20:14.457694",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "autumn night",
"corrected_text": "autumn night,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone it",
"corrected_text": "Saone, it",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "soddenwith",
"corrected_text": "soddened with",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:31.113524",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:58:15.839564",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late, dull autumn night",
"corrected_text": "A late, dull autumn night",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone it",
"corrected_text": "Saone, it",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:30.958463",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:07:19.357714",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "I hope so;",
"corrected_text": "I hoped so;",
"observed_line_number": 31,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The core intended change is converting \"hope\" to past tense \"hoped\" in the phrase \"I hope so\" \u2192 \"I hoped so\". The predicted edit captures this exact core change (\"I hope so;\" \u2192 \"I hoped so;\"). Although the prediction includes a semicolon as additional surrounding context, it does not alter the essential correction (hope \u2192 hoped). Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam,",
"corrected_text": "said Clennam; ",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:25.230950",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:21:09.525288",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "That's well! I hope so;",
"corrected_text": "That's well! I hoped so;",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:31.514463",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:59:06.858204",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "That's well! I hope so;",
"corrected_text": "That's well! I hoped so;",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:40.225952",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:08:17.157900",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both edits change the phrase \"hers and\" to \"hers, and\" by inserting a comma after \"hers\". The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters",
"corrected_text": "on hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The predicted edit also inserts a comma after \"hers\" (\"on hers matters\" \u2192 \"on hers, matters\"). Although the prediction includes extra surrounding context (the preceding word \"on\"), it preserves the same core punctuation change at the same location. Therefore, it correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change\u2014adding a comma after \"moment\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"beautiful\" in the phrase \"beautiful that\", yielding \"beautiful, that\". The core punctuation change is identical. Line number differences are ignored per instructions."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "they are to be married.",
"corrected_text": "They are to be married.",
"observed_line_number": 33,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth correction is to capitalize \"they\" to \"They\". The predicted edit applies the same core change\u2014capitalizing the initial \"they\"\u2014but includes additional surrounding context (\"are to be married.\"). This still captures the essential intended correction (\"they\" \u2192 \"They\").\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation, that papa had abandoned the idea.",
"corrected_text": "hesitation, that papa had abandoned the idea.",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "010"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:41.352957",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:21:59.516309",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth edit changes \"hers and\" to \"hers, and,\" which is functionally inserting a comma after \"hers\" before the conjunction \"and.\" The predicted edit changes \"hers\" to \"hers,\", which captures the same core punctuation change (adding the comma after \"hers\"). Although the prediction omits the following word \"and\" from its context, it still represents the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"hers matters\" to \"hers, matters\", which is specifically inserting a comma after the word \"hers\". The prediction changes \"hers\" to \"hers,\", which captures the same essential punctuation insertion at the same location. Although the prediction includes less surrounding context (it omits \"matters\"), it still reflects the core intended edit: adding a comma after \"hers\"."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"moment\" in the phrase \"moment with\" \u2192 \"moment, with.\" The predicted edit \"moment\" \u2192 \"moment,\" captures the same core punctuation change (adding a comma immediately after \"moment\"). Although it does not include the following word \"with,\" it still correctly represents the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The ground truth change is \"they\" \u2192 \"They\". The predicted edit makes the same core change, capitalizing the initial letter of the word. There is no discrepancy in original or corrected text content."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that papa had abandoned",
"corrected_text": "that papa had abandoned,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:46.278269",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T12:00:00.360384",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after the word \"hers\" in the phrase \"hers and\" to become \"hers, and.\" The predicted edit changes \"hers\" to \"hers,\", which captures the same essential punctuation insertion (a comma immediately after \"hers\"). Although it omits the following word \"and\" from the context, it still reflects the core intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 13,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"hers\" in the phrase \"hers matters\" (\"hers matters\" \u2192 \"hers, matters\"). The predicted edit also adds a comma immediately after \"hers\" (\"hers\" \u2192 \"hers,\"). Although the prediction includes less surrounding context (it omits \"matters\"), it captures the core intended punctuation change: inserting a comma after \"hers\". Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"moment\" in the phrase \"moment with\" \u2192 \"moment, with\". The predicted edit \"moment\" \u2192 \"moment,\" captures the same core punctuation change (adding a comma immediately after \"moment\"). Although it includes less surrounding context (does not include \"with\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth. Text Content Accuracy: Both ground truth and prediction change \"beautiful that\" to \"beautiful, that,\" inserting a comma after \"beautiful.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the types match exactly.\n\n2) Text Content Accuracy: The ground truth changes \"they\" to \"They\" (capitalizing the first letter). The predicted edit makes the exact same change from \"they\" to \"They\". This captures the core intended correction precisely.\n\nTherefore, the prediction is correct."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation, that papa had abandoned the idea.",
"corrected_text": "hesitation, that papa had abandoned the idea.",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "010"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:44.154411",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:09:17.041322",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 22,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"shore\" in the phrase \"shore and\" to make it \"shore, and.\" The original_text and corrected_text match the ground truth\u2019s core change exactly. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "within-doors",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "in",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "whatever",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:53.330613",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:22:51.378836",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 22,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change of inserting a comma after \"shore\". Line numbers are ignored as instructed. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "within-doors",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "in",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "whatever",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:29:53.359343",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T12:00:58.672444",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 22,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"shore\" in the phrase \"shore and\" to make it \"shore, and\". The original_text and corrected_text match the ground truth\u2019s intended punctuation change. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "within-doors",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that in our fancy,",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:30:12.332999",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:10:27.310011",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 3,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: The predicted type is \"capitalization,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The ground truth changes \"city\" to \"City.\" The predicted edit makes the same core change (\"city\" \u2192 \"City\") with identical original and corrected text. Since line numbers are ignored, the prediction captures the intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit adds the same comma after \"motionless\" in context (\"motionless when\" \u2192 \"motionless, when\"). Although the prediction includes the following word \"when\" as additional context, it captures the core intended punctuation change correctly."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them:",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "uniform and",
"corrected_text": "uniform, and",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,at",
"corrected_text": "existence, at",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it;to",
"corrected_text": "it; to",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,by",
"corrected_text": "view, by",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:30:02.106241",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:23:51.981023",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\nText Content Accuracy: The predicted change matches the ground truth core change, converting \"city\" to \"City\" with identical original and corrected text. No additional or conflicting modifications are introduced.\n\nTherefore, the prediction correctly captures the ground truth edit intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 12,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction changes \"motionless when\" to \"motionless, when\", which likewise inserts a comma immediately after \"motionless\". Although the prediction includes extra surrounding context (the following word \"when\"), it captures the same core punctuation change intended by the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them:",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it, to",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence, at",
"corrected_text": "existence at",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view, by",
"corrected_text": "view by",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:30:13.440315",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T12:02:02.606709",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"city\" to \"City\", capturing the identical core correction. No mismatch in original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with them;",
"corrected_text": "with them,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "uniform and",
"corrected_text": "uniform, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:30:05.888352",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:11:31.234009",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "snorts",
"corrected_text": "snorts.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:30:08.183797",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:24:39.934253",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\nText Content Accuracy: Both edits replace the exact same original text \"authorize\" with the corrected text \"authorise\". This captures the core intended change (American to British spelling). Line numbers are ignored as instructed, so the discrepancy there does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "a little",
"corrected_text": "a little,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "snorts",
"corrected_text": "snorts.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:30:19.242396",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T12:02:55.572468",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the same original word \"authorize\" with the same corrected word \"authorise\". This captures the core intended change (American to British spelling) with no discrepancy in the text content. Line number differences are to be ignored."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "of the driest",
"corrected_text": "of the driest,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "snorts",
"corrected_text": "snorts.",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:30:18.130359",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:12:23.651507",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For,Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me, not",
"corrected_text": "making me not",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:30:07.051996",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T11:25:27.971485",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt_4_1",
"date": "2026-02-22T15:30:16.058420",
"annotator": "GPT-4.1",
"annotation_date": "2025-04-14T12:03:53.275970",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": ",",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "k",
"corrected_text": "",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": ",",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2025-04-14",
"release_source": "https://openai.com/index/gpt-4-1/"
},
{
"model_name": "GPT-4.1 Mini",
"model_id": "gpt_4_1_mini",
"date": "2026-02-22T15:32:50.029870",
"shots": 2,
"config": {
"model_id": "gpt_4_1_mini",
"display_name": "GPT-4.1 Mini",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-14",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:14:56.517181",
"file_results": [
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:30:31.645469",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:14:56.517181",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "N",
"corrected_text": "N",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howlingover",
"corrected_text": "howling, over",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day, Sir",
"corrected_text": "to-day, Sir,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:30:50.499052",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:28:02.196555",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howlingover",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day, Sir",
"corrected_text": "to-day, sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:30:50.667183",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:06:36.915666",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howlingover",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day, Sir",
"corrected_text": "to-day, sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:30:36.855756",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:15:41.516526",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ",",
"corrected_text": "",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine to-",
"corrected_text": "cop question mark",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:30:40.520228",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:28:58.651072",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"No sir\" to \"No, sir,\" inserting a comma after \"No.\" This captures the core intended correction with no substantive difference.\n(Line number differences are ignored as instructed.)"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine to",
"corrected_text": "cap question mark",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:30:42.399882",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:07:36.119502",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 7,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"No\" in \"No sir\" to produce \"No, sir\". The original and corrected text strings match the ground truth\u2019s intended punctuation change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine to",
"corrected_text": "cop question mark",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:30:42.473995",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:16:28.945017",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "If",
"corrected_text": "If To cap",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sad",
"corrected_text": "asked",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:30:46.327256",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:30:00.570041",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "If",
"corrected_text": "If T cep",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sad",
"corrected_text": "asked",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:30:48.566494",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:08:25.521938",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "If",
"corrected_text": "If T cap",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sad",
"corrected_text": "asked",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:12.659297",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:17:17.895145",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ",",
"corrected_text": "",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "tion did",
"corrected_text": "tion did not",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:09.058420",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:30:54.880709",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "tion did",
"corrected_text": "tion did not",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ",",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:08.637000",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:09:25.316912",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "and",
"corrected_text": "",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "did",
"corrected_text": "did",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "earnestly",
"corrected_text": "earnestly,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:10.577351",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:18:09.381172",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:05.370272",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:31:41.847784",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:10.795065",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:10:19.091682",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:25.554399",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:19:03.398451",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 3,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Saone\") and the same corrected text (\"Sa\u00f4ne\"), capturing the identical core change (adding the circumflex on the 'o'). Line numbers are to be ignored, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 8,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\nText Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\", correctly adding the circumflex accent over the \"o\". This captures the core intended correction with identical original and corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 13,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction are labeled as a \"replacement,\" so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"sodden\" to \"soddened.\" The prediction changes \"sodden with\" to \"soddened with,\" which includes extra surrounding context (\"with\") but preserves the same core correction (\"sodden\" \u2192 \"soddened\"). Therefore, the predicted edit captures the intended textual change."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "A",
"corrected_text": "A",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "stream, like",
"corrected_text": "stream like",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "autumn",
"corrected_text": "",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "One man slowly moving on towards Chalons was",
"corrected_text": "One man slowly moving on towards Chalons, was",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:30.394319",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:32:36.122665",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 13,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2. Text Content Accuracy: The ground truth changes \"sodden\" to \"soddened\". The prediction changes \"sodden with\" to \"soddened with\", which includes extra surrounding context but preserves the same core correction (adding \"ed\" to \"sodden\"). Therefore, the essential intended edit is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "stream, like",
"corrected_text": "stream like",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "autumn",
"corrected_text": "",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "a",
"corrected_text": "A",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "One man slowly moving on towards Chalons was",
"corrected_text": "One man slowly moving on towards Chalons, was",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:15.409496",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:11:21.818123",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the original text \"Saone\" with the corrected text \"Sa\u00f4ne\" (adding the circumflex over 'o'). The core change is identical in the prediction and ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "On the banks of the river Saone it was wet",
"corrected_text": "On the banks of the river Sa\u00f4ne it was wet",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "One man slowly moving on towards Chalons was",
"corrected_text": "One man slowly moving on towards Chalons, was",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with wet",
"corrected_text": "sodden with",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "A late, dull autumn night",
"corrected_text": "A late, dull autumn night",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:33.698498",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:19:58.564313",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 7,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text \"heart so\" and the same corrected text \"heart, so\", capturing the identical core change (inserting a comma after \"heart\"). Ignoring line numbers, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "x",
"corrected_text": "",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "tially up into his face, with some hurried words to the effect that she thanked him from her heart (as",
"corrected_text": "indeed tially up into his face, with some hurried words to the effect that she thanked him from her heart (as",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hoped so",
"corrected_text": "I hoped so",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:36.024838",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:33:28.941008",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "his own",
"corrected_text": "his, own",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "generous",
"corrected_text": "generous,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "face, with some hurried words to the effect that she thanked him from her heart (as",
"corrected_text": "face, indeed, with some hurried words to the effect that she thanked him from her heart (as",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hoped so",
"corrected_text": "I hoped so",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart; so",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:35.023080",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:12:39.334941",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 7,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"heart so\") and the same correction (\"heart, so\"), capturing the identical core change\u2014adding a comma after \"heart\". Line numbers are ignored as instructed, so there is no discrepancy affecting correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "his own",
"corrected_text": "his, own",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "generous Mr.",
"corrected_text": "generous",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "face, with some hurried words to the effect that she thanked him from her heart (as",
"corrected_text": "face, indeed with some hurried words to the effect that she thanked him from her heart (as",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hoped so",
"corrected_text": "I hoped so",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:32.665267",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:20:48.307732",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers) matters",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "strikingly",
"corrected_text": "strikingly",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation, that papa had abandoned",
"corrected_text": "hesitation, that papa had abandoned",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:38.419364",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:34:27.156694",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same textual change: inserting a comma after \"moment\" in the phrase \"moment with\", resulting in \"moment, with\". The original_text and corrected_text match the ground truth's core change exactly.\n\nTherefore, the predicted edit correctly captures the ground truth intention (ignoring line numbers as instructed)."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers) matters",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "they are to be married.",
"corrected_text": "They are to be married.",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:42.039351",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:13:33.793539",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers) matters",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "strikingly",
"corrected_text": "strikingly",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:31:51.583157",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:21:48.005149",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"Thank you\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "shoreand",
"corrected_text": "shore, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "y",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "x",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:03.690945",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:35:24.267534",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shoreand",
"corrected_text": "shore, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "y",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "x",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:05.906954",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:14:29.862283",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"Thank you\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "y",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "y",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:33.034964",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:22:45.825196",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "ity",
"corrected_text": "City",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "reluctant return",
"corrected_text": "reluctant return to",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "busy existence,at",
"corrected_text": "busy existence, at",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "it;to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "motionlesswhen",
"corrected_text": "motionless when",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "stand-still;to",
"corrected_text": "stand-still; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "view,by",
"corrected_text": "view, by",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence;is",
"corrected_text": "existence; is",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "liege lord",
"corrected_text": "liege lord",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:50.029870",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:36:21.469053",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to add a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same essential punctuation change, adding a comma immediately after \"view\" within a slightly longer context (\"view by\" \u2192 \"view, by\"). This captures the core change accurately, with only additional surrounding text included."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "ity",
"corrected_text": "Ity",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "return",
"corrected_text": "to return",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence at",
"corrected_text": "existence; at",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless; when",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "liege lord",
"corrected_text": "liege lord",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:39.226647",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:15:27.326814",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "ity",
"corrected_text": "City",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "reluctant return",
"corrected_text": "reluctant to return",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "busy existence at",
"corrected_text": "busy existence, at",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "it; to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless; when",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "liege lord",
"corrected_text": "liege lord",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:11.593642",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:23:38.423016",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that boiling-over",
"corrected_text": "that boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorife",
"corrected_text": "authorise",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest of snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:14.499191",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:37:16.155440",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that boiling-over",
"corrected_text": "that boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorife",
"corrected_text": "authorise",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest of snorts.",
"corrected_text": "driest of snorts.",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:17.194626",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:16:23.974648",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that boiling-over",
"corrected_text": "the boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorife",
"corrected_text": "authorise",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest of snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:23.241537",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:24:29.943396",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me,",
"corrected_text": "me, a",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "x",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:32.275183",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T11:38:08.155635",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "x",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt_4_1_mini",
"date": "2026-02-22T15:32:37.982892",
"annotator": "GPT-4.1 Mini",
"annotation_date": "2025-04-14T12:17:14.113807",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "x",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me,",
"corrected_text": "me a",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2025-04-14",
"release_source": "https://openai.com/index/gpt-4-1/"
},
{
"model_name": "GPT-4.1 Nano",
"model_id": "gpt_4_1_nano",
"date": "2026-02-22T16:21:14.272664",
"shots": 2,
"config": {
"model_id": "gpt_4_1_nano",
"display_name": "GPT-4.1 Nano",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-14",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:27:05.347949",
"file_results": [
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:32:38.686905",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:27:05.347949",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and rattling his money at it, apostrophised it with a short laugh",
"corrected_text": "and rattling his money at it, apostrophised it with a short laugh.",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:32:41.376425",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:40:43.048341",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and rattling his money at it, apostrophised it with a short laugh",
"corrected_text": "and rattling his money at it, apostrophised it with a short laugh.",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:32:45.759470",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:19:58.783197",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and rattling his money at it, apostrophised it with a short laugh",
"corrected_text": "and rattling his money at it, apostrophised it with a short laugh.",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:32:54.836888",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:27:51.606915",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cap",
"corrected_text": "cap",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "cap",
"corrected_text": "a cushion",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "a cushion",
"corrected_text": "and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "mark",
"corrected_text": "mask",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:32:59.783121",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:41:44.160907",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "w",
"corrected_text": "Why",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "w",
"corrected_text": "why",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "cap",
"corrected_text": "cap",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "cap",
"corrected_text": "a cushion",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "mask",
"corrected_text": "mask",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:00.382610",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:20:54.251713",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "w",
"corrected_text": "Why",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "w",
"corrected_text": "why",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "cap",
"corrected_text": "cap",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "a cushion",
"corrected_text": "a cushion",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "mark",
"corrected_text": "mask",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T16:21:07.610525",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:28:49.986518",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "If he",
"corrected_text": "If he",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "asked",
"corrected_text": "asked",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "said",
"corrected_text": "said",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "Ce",
"corrected_text": "Ce",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T16:21:05.188274",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:42:32.690693",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "he",
"corrected_text": "If he",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "asked",
"corrected_text": "asked",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "\"True, sir, true to a certain extent. But what is a man to do?\"",
"corrected_text": "sir, true to a certain extent. But what is a man to do?",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "say",
"corrected_text": "said",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T16:21:05.385458",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:22:00.893603",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "If he",
"corrected_text": "If he",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "asked",
"corrected_text": "asked",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "say",
"corrected_text": "said",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "Ce",
"corrected_text": "Ce",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:21.209415",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:29:47.230814",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "telling him not to be uneasy on her, for she was quite well; but he had not seen her, for what, in their intercourse, was a long time.",
"corrected_text": "telling him not to be uneasy on her behalf, for she was quite well; but he had not seen her, for what, in their intercourse, was a long time.",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam!\u2014Tattyoram!",
"corrected_text": "Clennam!\u2014Tattyoram!",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "What\u2019s the matter?",
"corrected_text": "What\u2019s the matter?",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Never to come back,\u201d",
"corrected_text": "\u201cNever to come back,\u201d",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "How did it happen? Pray sit down and tell me.",
"corrected_text": "\u201cHow did it happen? Pray sit down and tell me.\u201d",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "costing",
"corrected_text": "costing",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:25.672236",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:43:42.824742",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "telling him not to be uneasy on her behalf, for she was quite well; but he had not seen her, for what, in their intercourse, was a long time.",
"corrected_text": "telling him not to be uneasy on her behalf, for she was quite well; but he had not seen her, for what, in their intercourse, was a long time.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam!\u2014Tattyoram!",
"corrected_text": "Clennam!\u2014Tattyoram!",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "What\u2019s the matter?",
"corrected_text": "What\u2019s the matter?",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Never to come back,\u201d said Mr. Meagles, shaking his head. \u201cYou don\u2019t know that girl\u2019s passionate and proud character. A team of horses couldn\u2019t draw her back now; the bolts and bars of the old Bastille couldn\u2019t keep her.\u201d",
"corrected_text": "\u201cNever to come back,\u201d said Mr. Meagles, shaking his head. \u201cYou don\u2019t know that girl\u2019s passionate and proud character. A team of horses couldn\u2019t draw her back now; the bolts and bars of the old Bastille couldn\u2019t keep her.\u201d",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "costing",
"corrected_text": "costing",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:25.722713",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:23:05.744461",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "costing absolu-",
"corrected_text": "costing absolutely nothing.",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and, telling him not to be uneasy on her behalf, for she was quite well; but he had not seen her, for what, in their intercourse, was a long time.",
"corrected_text": "and, telling him not to be uneasy on her behalf, for she was quite well; but he had not seen her, for what, in their intercourse, was a long time.",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam!\u2014Tattycoram!",
"corrected_text": "Clennam!\u2014Tattycoram!",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cWhat\u2019s the matter?\u201d",
"corrected_text": "\u201cWhat\u2019s the matter?\u201d",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cLost!\u201d",
"corrected_text": "\u201cLost!\u201d",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cWhy, bless my heart alive!\u201d cried Clennam in amazement. \u201cWhat do you mean?\u201d",
"corrected_text": "\u201cWhy, bless my heart alive!\u201d cried Clennam, in amazement. \u201cWhat do you mean?\u201d",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cWouldn\u2019t count five-and-twenty, sir; couldn\u2019t be got to do it; stopped at eight, and took herself off.\u201d",
"corrected_text": "\u201cWouldn\u2019t count five-and-twenty, sir; couldn\u2019t be got to do it; stopped at eight, and took herself off.\u201d",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cLeft your house?\u201d",
"corrected_text": "\u201cLeft your house?\u201d",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cNever to come back,\u201d said Mr. Meagles, shaking his head. \u201cYou don\u2019t know that girl\u2019s passionate and proud character. A team of horses couldn\u2019t draw her back now; the bolts and bars of the old Bastille couldn\u2019t keep her.\u201d",
"corrected_text": "\u201cNever to come back,\u201d said Mr. Meagles, shaking his head. \u201cYou don\u2019t know that girl\u2019s passionate and proud character. A team of horses couldn\u2019t draw her back now; the bolts and bars of the old Bastille couldn\u2019t keep her.\u201d",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:29.584450",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:30:33.341156",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "THIRTY",
"corrected_text": "Thirty",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow.",
"corrected_text": "Period here?",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:30.167999",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:44:31.783050",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Hirty",
"corrected_text": "Thirty",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "strangers",
"corrected_text": "stranger",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "univer-sal",
"corrected_text": "universal",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow.",
"corrected_text": "Period here?",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:29.863616",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:23:57.857369",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Hirty",
"corrected_text": "Thirty",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "strangers",
"corrected_text": "stranger",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "univer-sal",
"corrected_text": "universal",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow.",
"corrected_text": "Period here?",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:49.458138",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:31:27.461349",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "a sullied looking-glass in a gloomy place, reflected",
"corrected_text": "like a sullied looking-glass in a gloomy place, reflected",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "heavy streak, occasionally made a little ragged by a row of trees",
"corrected_text": "heavy streak, occasionally made a little ragged by a row of poplar trees",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and his shoes and gaiters trodden out, his hair and beard untrimmed; the cloak he carried over his shoulder, and the clothes he wore, sodden with wet; limping along in pain and difficulty; he looked as if the clouds were hurrying from him, as if the wail of the wind and the shuddering of the grass",
"corrected_text": "ed,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "the low mysterious plashing of the water murmured at him, as if the fitful autumn night were disturbed by him.",
"corrected_text": "the low mysterious plashing of the water murmured at him, as if the fitful autumn night were disturbed by him.",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late, dull autumn night was closing in upon the river Saone.",
"corrected_text": "A late, dull autumn night was closing in upon the river Saone.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "One man slowly moving on towards Chalons was the only visible",
"corrected_text": "One man, slowly moving on towards Chalons, was the only visible",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:53.869475",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:45:23.973987",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "One man slowly moving on towards Chalons, was the only visible",
"corrected_text": "One man, slowly moving on towards Chalons, was the only visible",
"observed_line_number": 8,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"man slowly\" to \"man, slowly\" (inserting a comma after \"man\"). The prediction makes the same core change within a longer span: \"One man slowly ...\" to \"One man, slowly ...\". Although the prediction includes additional surrounding context, it correctly captures the essential intended punctuation insertion."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "like a sullied looking-glass in a gloomy place, reflected",
"corrected_text": "like a sullied looking-glass in a gloomy place, reflected",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ed",
"corrected_text": "ed,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with wet; limping",
"corrected_text": "with wet; limping",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "autumn night were disturbed by him.",
"corrected_text": "autumn night were disturbed by him.",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "late, dull autumn night was closing in upon the river Saone.",
"corrected_text": "A late, dull autumn night was closing in upon the river Saone.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heavy streak, occasionally made a little ragged by a row of poplar trees",
"corrected_text": "heavy streak, occasionally made a little ragged by a row of poplar trees",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:53.334509",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:24:48.800003",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "trees against the wrathful sunset.",
"corrected_text": "trees, hyphen",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "he limped on again, toiling and muttering.",
"corrected_text": "then",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "A late, dull autumn night was closing in upon the river Saone.",
"corrected_text": "A late, dull autumn night was closing in upon the river Saone.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cloaked he carried",
"corrected_text": "ed",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:50.064992",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:32:17.930245",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thank him from her heart (as",
"corrected_text": "thank him from her heart (as",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "she did, if it be the source of earnestness),",
"corrected_text": "she did, if it be the source of earnestness),",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "and then a word of en- couragement from him, as they walked on slowly and almost silently under the darkening trees.",
"corrected_text": "and now and then a word of encouragement from him, as they walked on slowly and almost silently under the darkening trees.",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cAnd, now, Minnie Gowan,\u201d at length, said Clennam, smiling; \u201cwill you ask me nothing?\u201d",
"corrected_text": "\u201cAnd, now, Minnie Gowan,\u201d at length, said Clennam, smiling; \u201cwill you ask me nothing?\u201d",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cOh! I have very much to ask of you.\u201d",
"corrected_text": "\u201cOh! I have very much to ask of you.\u201d",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:50.065601",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:46:17.792014",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thank him from her heart (as",
"corrected_text": "thank him from her heart (as",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "she did, if it be the source of earnestness),",
"corrected_text": "she did, if it be the source of earnestness),",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "And now, Minnie Gowan,",
"corrected_text": "and, now, Minnie Gowan,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "smiling; \"will",
"corrected_text": "smiling; \"will",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"Oh! I have very much to ask of you.",
"corrected_text": "\"Oh! I have very much to ask of you.\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:33:50.898513",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:25:50.112833",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thank him from her heart (as",
"corrected_text": "thank him from her heart (as",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "she did, if it be the source of earnestness),",
"corrected_text": "she did, if it be the source of earnestness),",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "and, if it be the source of earnestness),",
"corrected_text": "and, now and then a word of encouragement from him,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling, \"will you ask me nothing?\"",
"corrected_text": "smiling; \"will you ask me nothing?\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so; I am not disappointed.\"",
"corrected_text": "so; I am not disappointed.\"",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:07.363245",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:33:22.319906",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and thanked her",
"corrected_text": "and thanked her.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with a look in which regard for him and trustfulness in him were strikingly blended with a kind of timid sorrow for him, she was so beautiful that it was well for his peace\u2014or ill for his peace, he did not quite know which \u2014 that he had made that vigorous resolution he had so often thought about.",
"corrected_text": "with a look in which regard for him and trustfulness in him were strikingly blended with a kind of timid sorrow for him, she was so beautiful that it was well for his peace\u2014or ill for his peace, he did not quite know which \u2014 that he had made that vigorous resolution he had so often thought about.",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "At this, he thought directly, \"they are to be married.\"",
"corrected_text": "At this, he thought directly, \"they are to be married.\"",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:07.001243",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:47:17.953275",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and thanked her",
"corrected_text": "and thanked her.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "with a look in which regard for him and trustfulness in him were strikingly blended with a kind of timid sorrow for him, she was so beautiful that it was well for his peace\u2014or ill for his peace, he did not quite know which",
"corrected_text": "with a look in which regard for him and trustfulness in him were strikingly blended with a kind of timid sorrow for him, she was so beautiful that it was well for his peace\u2014or ill for his peace, he did not quite know which\u2014that he had made that vigorous resolution he had so often thought about.",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "At this, he thought directly, \"they are to be married.\"",
"corrected_text": "At this, he thought directly, \"they are to be married.\"",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:07.003642",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:26:44.463524",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and thanked her",
"corrected_text": "and thanked her.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "with a look in which regard for him and trustfulness in him were strikingly blended with a kind of timid sorrow for him, she was so beautiful that it was well for his peace\u2014or ill for his peace, he did not quite know which",
"corrected_text": "with a look in which regard for him and trustfulness in him were strikingly blended with a kind of timid sorrow for him, she was so beautiful that it was well for his peace\u2014or ill for his peace, he did not quite know which\u2014that he had made that vigorous resolution he had so often thought about.",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "At this, he thought directly, \"they are to be married.\"",
"corrected_text": "At this, he thought directly, \"they are to be married.\"",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:17.724306",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:34:16.886643",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "in our fancy",
"corrected_text": "in our fancy,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:07.362010",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:48:06.456076",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "in our fancy",
"corrected_text": "in our fancy,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "gently launched them on the flowing river.",
"corrected_text": "gently launched them on the flowing river.",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that once were",
"corrected_text": "that once were",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:18.412166",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:27:43.652327",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "in our fancy",
"corrected_text": "in our fancy,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:28.751124",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:35:11.664373",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "lucrant return of the same sequences of machinery, like a dragging piece of clockwork.",
"corrected_text": "lucrant return of the same sequences of machinery, like a dragging piece of clockwork.",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "images of people as they too used to be, with little or no allowance made for the lapse of time since they were seen; of these, there must have been many in the long routine of gloomy days. To stop the clock of busy existence, at the hour when we were personally sequestered from it; to suppose mankind stricken motionless when we were brought to a stand-still; to be unable to measure the changes beyond our view, by any larger standard than the shrunken one of our own uniform and contracted existence; is the infirmity of many invalids, and the mental unhealthiness of almost all recluses.",
"corrected_text": "images of people as they too used to be, with little or no allowance made for the lapse of time since they were seen; of these, there must have been many in the long routine of gloomy days. To stop the clock of busy existence, at the hour when we were personally sequestered from it; to suppose mankind stricken motionless when we were brought to a stand-still; to be unable to measure the changes beyond our view, by any larger standard than the shrunken one of our own uniform and contracted existence; is the infirmity of many invalids, and the mental unhealthiness of almost all recluses.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "lord",
"corrected_text": "lord",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to each recurring with its accompanying monotony, always the same re-",
"corrected_text": "to each recurring with its accompanying monotony, always the same re-",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:30.439155",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:49:02.309128",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "busy",
"corrected_text": "at",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "a stand-still",
"corrected_text": "; to",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "unhealthiness",
"corrected_text": "; is",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liege",
"corrected_text": "lord",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "city",
"corrected_text": "city",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each",
"corrected_text": "to",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:34.827731",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:28:49.010351",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of the same sequences of machinery, like a dragging piece of clockwork.",
"corrected_text": "of the same sequences of machinery, like a dragging piece of clockwork.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "lord",
"corrected_text": "lord",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Pictures of demolished streets and altered houses, as they formerly were when the occupant of the chair was familiar with them; images of people as they too used to be, with little or no allowance made for the lapse of time since they were seen; of these, there must have been many in the long routine of gloomy days. To stop the clock of busy existence, at the hour when we were personally sequestered from it; to suppose mankind stricken motionless when we were brought to a stand-still; to be unable to measure the changes beyond our view, by any larger standard than the shrunken one of our own uniform and contracted existence; is the infirmity of many invalids, and the mental unhealthiness of almost all recluses.",
"corrected_text": "Pictures of demolished streets and altered houses, as they formerly were when the occupant of the chair was familiar with them; images of people as they too used to be, with little or no allowance made for the lapse of time since they were seen; of these, there must have been many in the long routine of gloomy days. To stop the clock of busy existence, at the hour when we were personally sequestered from it; to suppose mankind stricken motionless when we were brought to a stand-still; to be unable to measure the changes beyond our view, by any larger standard than the shrunken one of our own uniform and contracted existence; is the infirmity of many invalids, and the mental unhealthiness of almost all recluses.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T16:21:14.272664",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:36:05.994062",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "",
"corrected_text": "him.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "",
"corrected_text": "authorise",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "",
"corrected_text": "driest of snorts.",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "",
"corrected_text": "boiling-over",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T16:21:11.706105",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:50:04.320409",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "old Christian,",
"corrected_text": "boiling-over",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "recompense",
"corrected_text": "recompense",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "authorise",
"corrected_text": "authorise",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts.",
"corrected_text": "driest of snorts.",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T16:21:13.249682",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:29:45.778495",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "authorise",
"corrected_text": "authorise",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": null,
"original_text": "snorts.",
"corrected_text": "driest of snorts.",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:33.658777",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:36:51.332489",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room,",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that is past and what is past can never be recalled except in his own",
"corrected_text": "that is past and what is past can never be recalled except in his own",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:33.670897",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T11:50:57.761452",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room,",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that is past and what is past can never be recalled except in his own case as poor Mr. F. said when he was in spirits Cucumber and therefore never ate it.",
"corrected_text": "that is past and what is past can never be recalled except in his own case as poor Mr. F. said when he was in spirits Cucumber and therefore never ate it.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt_4_1_nano",
"date": "2026-02-22T15:34:45.018700",
"annotator": "GPT-4.1 Nano",
"annotation_date": "2025-04-14T12:30:39.969311",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room,",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that is past and what is past can never be recalled except in his own case as poor Mr. F. said when he was in spirits Cucumber and therefore never ate it.",
"corrected_text": "that is past and what is past can never be recalled except in his own case as poor Mr. F. said when he was in spirits Cucumber and therefore never ate it.",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2025-04-14",
"release_source": "https://openai.com/index/gpt-4-1/"
},
{
"model_name": "GPT-4.5 Preview",
"model_id": "gpt-4.5-preview",
"date": "2026-02-22T15:24:01.384481",
"shots": 2,
"config": {
"model_id": "gpt-4.5-preview",
"display_name": "GPT-4.5 Preview",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:04:52.602883",
"file_results": [
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:21:39.236789",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:04:52.602883",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to make it \"pockets, and\". The original_text and corrected_text match the ground truth\u2019s intended punctuation change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "day sir",
"corrected_text": "day, sir",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:21:43.773877",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:27:36.810059",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "day sir",
"corrected_text": "day, sir",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:21:45.954087",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:24:43.426613",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" so that \"pockets and\" becomes \"pockets, and\". The original and corrected text strings match the ground truth intention. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "howling over yonder to-day",
"corrected_text": "howling, over yonder to-day",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:21:59.575153",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:05:52.284055",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "why did he dine",
"corrected_text": "why did he dine?",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:00.079260",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:28:24.174562",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "why did he dine",
"corrected_text": "why did he dine?",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:08.489603",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:25:58.998990",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word of the phrase (\"why did he dine\" \u2192 \"Why did he dine\"), i.e., changing \"why\" to \"Why\". The prediction captures this core change directly (\"why\" \u2192 \"Why\"). Although it includes less surrounding context, it still represents the same essential capitalization correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones!\"",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:12.076504",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:06:54.378430",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "said",
"corrected_text": "",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:11.715695",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:29:11.606975",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "said",
"corrected_text": "",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:12.315879",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:27:31.795595",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "said",
"corrected_text": "",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:37.417362",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:08:19.268198",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "tion",
"corrected_text": "tion;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "n",
"corrected_text": "Clennam",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "tion,",
"corrected_text": "",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "earnestly",
"corrected_text": "earnestly,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "comma?",
"corrected_text": ",",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:39.052901",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:30:16.721324",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "did",
"corrected_text": ", did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "for what",
"corrected_text": ", for what,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "tion,",
"corrected_text": "",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "telling",
"corrected_text": ", telling",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "said (comma?)",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:41.954618",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:29:58.656455",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "for what",
"corrected_text": ", for what,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "amazement",
"corrected_text": ", in amazement",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "tion,",
"corrected_text": "",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "telling",
"corrected_text": ", telling",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "said (comma?)",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Clennam,",
"corrected_text": "Clennam",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:26.935686",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:09:14.921923",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the same core change with identical original and corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter 1",
"corrected_text": "Chapter 1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:27.023038",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:31:14.318667",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. The core punctuation correction is identical. Line numbers are ignored as instructed, so the discrepancy there does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter 1",
"corrected_text": "Chapter 1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:41.690394",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:31:06.429505",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter 1",
"corrected_text": "Chapter 1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:50.976855",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:10:25.831971",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original_text (\"man slowly\") and the same corrected_text (\"man, slowly\"), capturing the identical core change: insertion of a comma after \"man\". Ignoring line numbers as instructed, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "like a sullied",
"corrected_text": ", like a sullied",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late, dull autumn",
"corrected_text": "A late, dull autumn",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Saone,",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:50.110819",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:32:18.551991",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late dull autumn",
"corrected_text": "late, dull autumn",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Saone,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:59.825205",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:32:59.539359",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 11,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"man slowly\" to \"man, slowly,\" correctly capturing the intended insertion of a comma after \"man.\" Line numbers are to be ignored, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 11,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"Chalons\" in the phrase \"Chalons was\" to become \"Chalons, was\". The original and corrected text strings match the ground truth intention. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes \"sodden\" to \"soddened\". The predicted edit changes \"sodden with\" to \"soddened with\", which includes extra surrounding context but preserves the same core substitution (sodden \u2192 soddened). Therefore, the essential intended correction is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "stream, like",
"corrected_text": "stream,^ like",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "dull,",
"corrected_text": "",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone.",
"corrected_text": "Saone.^ The",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone it",
"corrected_text": "Saone^ it",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:45.466586",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:11:26.271037",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "up into his face, with some hurried words",
"corrected_text": "up into his face, indeed, with some hurried words",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "thanked him from her heart",
"corrected_text": "thanked him from her heart,",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hoped",
"corrected_text": "I hoped so",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "so",
"corrected_text": "",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:22:41.232663",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:33:21.832504",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "up into his face, with some",
"corrected_text": "up into his face, indeed, with some",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "thanked him from her heart",
"corrected_text": "thanked him with her heart",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "so;",
"corrected_text": ";",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "That's well!",
"corrected_text": "That's well! I hoped so;",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:06.763371",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:34:35.408086",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 28,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"length said\") and the same corrected text (\"length, said\"), capturing the core change of inserting a comma after \"length\". Since the essential punctuation change matches, the prediction aligns with the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "totally up into his face, with some",
"corrected_text": "totally up into his face, indeed, with some",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "thanked him from her heart",
"corrected_text": "thanked him with her heart",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "first",
"corrected_text": "",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "That's well! I hoped so;",
"corrected_text": "That's well! I hoped so;",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:09.000173",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:12:30.866074",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change, inserting a comma after \"hers\" in the phrase \"hers and\" to make it \"hers, and\". The original and corrected text strings match the ground truth\u2019s intention exactly. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both apply the same core change: inserting a comma after \"hers\" to turn \"hers matters\" into \"hers, matters\". The original and corrected text content match the ground truth intention. Line numbers are ignored per instructions."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change: insertion of a comma after \"moment.\" Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same correction (\"beautiful, that\"), inserting a comma after \"beautiful\". The core change is identical."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation, that papa had abandoned",
"corrected_text": "hesitation, that papa had abandoned,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "he thought directly,",
"corrected_text": "",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:10.427853",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:34:39.015913",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 16,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both the ground truth and predicted edit are labeled as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both edits change the exact same text from \"hers and\" to \"hers, and,\" inserting a comma after \"hers.\" This captures the core intended correction with no mismatch in the essential change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 18,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers.\" The original and corrected text match the ground truth\u2019s core change with no discrepancies. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"moment with\" to \"moment, with\", inserting a comma after \"moment\". This captures the core intended punctuation correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"beautiful that\" to \"beautiful, that,\" inserting a comma after \"beautiful.\" This captures the core intended punctuation correction with identical original and corrected text content.\n\nLine numbers are ignored as instructed, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation, that papa had abandoned",
"corrected_text": "hesitation, that papa had abandoned,",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "he thought directly,",
"corrected_text": "",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:10.908984",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:36:18.779202",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers\". Line numbers are ignored as instructed, so this does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both ground truth and prediction change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers\". This captures the core intended correction with identical original and corrected text.\n\nTherefore, the predicted edit correctly matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change\u2014adding a comma after \"moment\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"beautiful\" in the phrase \"beautiful that,\" resulting in \"beautiful, that.\" The original and corrected texts match the ground truth intention with no discrepancies."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hesitation, that papa had abandoned",
"corrected_text": "(Cap) hesitation, that papa had abandoned",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "he thought directly,",
"corrected_text": "",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:12.074406",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:13:38.779387",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 28,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original_text (\"shore and\") and the same corrected_text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Ignoring line numbers as instructed, the prediction matches the ground truth intent."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you?\"",
"corrected_text": "you,\"",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "once",
"corrected_text": "than",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:18.672166",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:35:41.900561",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Ignoring line numbers, the prediction matches the ground truth\u2019s intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "once",
"corrected_text": "than",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:26.258170",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:38:16.454087",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the core intended change\u2014adding a comma after \"shore\". Line numbers are ignored, so this does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\", capturing the same core change. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moonlight",
"corrected_text": "moonlight,",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "flowers pale",
"corrected_text": "flowers, pale",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moonlight",
"corrected_text": "moonlight,",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:47.672276",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:15:04.995713",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a period to the end of \"Chapter XXIX\" to make it \"Chapter XXIX.\". The original and corrected texts are identical between ground truth and prediction, so the intended punctuation correction is captured precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 4,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\nText Content Accuracy: Both specify the same core transformation: \"city\" \u2192 \"City\". This captures the essential intended correction. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "lord",
"corrected_text": "",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "of",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "still",
"corrected_text": "still;",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:40.387430",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:37:22.607740",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 4,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction specify the same core change: \"city\" \u2192 \"City\". This captures the essential intended correction with no discrepancy in the text being changed.\n\nLine numbers are ignored as instructed, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "xxix",
"corrected_text": "xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "still",
"corrected_text": "still;",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "l",
"corrected_text": "",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "of",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "existence",
"corrected_text": "existence at",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:37.987259",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:40:45.926778",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of",
"corrected_text": "return of",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "lord",
"corrected_text": "liege",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:50.712522",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:16:13.515060",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts \"of\" so that \"dryest snorts\" becomes \"driest of snorts\" (within the larger phrase \"of the \u2026 .\"). The predicted edit makes the same essential change: \"dryest snorts\" \u2192 \"driest of snorts\". Although the prediction omits surrounding context (\"of the\" and the final period), it captures the core intended correction (spelling change + insertion of \"of\")."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "been a little premature",
"corrected_text": "been a little premature.",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "thropic",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a boiling-",
"corrected_text": "",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:50.787206",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:38:27.742295",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both replace the same original text \"authorize\" with the same corrected text \"authorise\". The core intended change (American to British spelling) is captured exactly. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "thropic",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a",
"corrected_text": "",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:54.278144",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:42:41.441061",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts \"of\" so the phrase becomes \"driest of snorts\" (within the larger context \"of the ... .\"). The predicted edit also changes \"dryest snorts\" to \"driest of snorts\", capturing the same core correction (spelling correction + insertion of \"of\"). Although the prediction omits surrounding context (\"of the\" and the final period), it correctly represents the essential intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 34,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n\nText Content Accuracy: Both edits replace the same original text (\"authorize\") with the same corrected text (\"authorise\"). This captures the identical core change (American to British spelling). Ignoring line numbers as instructed, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "thropic old buck.",
"corrected_text": "thropic old buck",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a",
"corrected_text": "",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:55.273845",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T20:17:19.562803",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "For",
"corrected_text": "For,",
"observed_line_number": 3,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma immediately after the word \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The predicted edit \"For\" \u2192 \"For,\" captures the same core punctuation change (adding a comma after \"For\"), even though it provides less surrounding context. This still matches the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed-",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "me",
"corrected_text": "me,",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:23:57.751607",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-07T22:39:27.739098",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "For",
"corrected_text": "For,",
"observed_line_number": 3,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The predicted edit \"For\" \u2192 \"For,\" captures the same core punctuation change (adding a comma immediately after \"For\"). Although it omits the surrounding context (\"Mr. Pancks\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed-",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "me,",
"corrected_text": "",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-4.5-preview",
"date": "2026-02-22T15:24:01.384481",
"annotator": "GPT-4.5 Preview",
"annotation_date": "2025-04-08T11:44:06.388430",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "For",
"corrected_text": "For,",
"observed_line_number": 3,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The predicted edit captures the same essential change by converting \"For\" \u2192 \"For,\", even though it provides less surrounding context. This still matches the core punctuation insertion intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed-",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2025-02-27",
"release_source": "https://openai.com/index/introducing-gpt-4-5/"
},
{
"model_name": "GPT-4.5 Preview (OR)",
"model_id": "or_gpt_4_5_preview",
"date": "2026-02-22T15:50:06.897959",
"shots": 2,
"config": {
"model_id": "or_gpt_4_5_preview",
"display_name": "GPT-4.5 Preview (OR)",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:26:38.049783",
"file_results": [
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:47:59.153582",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:26:38.049783",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "day sir",
"corrected_text": "day, sir",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:47:59.154041",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:45:22.335965",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "day sir",
"corrected_text": "day, sir",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:08.920867",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:29:56.521621",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original phrase \"howling over\" and the same corrected form \"howling, over\", inserting a comma after \"howling\". This captures the core intended punctuation change with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both edits change \"yonder to-day\" to \"yonder, to-day\" by inserting a comma after \"yonder\". The original and corrected text strings are identical between ground truth and prediction, capturing the same core punctuation change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"pockets and\" to \"pockets, and\" by inserting a comma after \"pockets\". The core punctuation change is identical, with no discrepancies in the original or corrected text. Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:13.397214",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:27:25.927128",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "think why",
"corrected_text": "think, why",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones!\"",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:07.265964",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:46:10.638574",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "why did he dine",
"corrected_text": "why did he dine?",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:08.494512",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:30:54.584505",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 25,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the first word of the phrase, changing \"why did he dine\" to \"Why did he dine\" (i.e., \"why\" \u2192 \"Why\"). The prediction captures this core change by changing \"why\" to \"Why\". Although it includes less surrounding context than the ground truth, it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones!\"",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:16.778407",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:28:15.883995",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "asked",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "he",
"corrected_text": "He",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "said",
"corrected_text": "",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:16.053373",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:31:53.050149",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "asked",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "said",
"corrected_text": "",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:42.502379",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:29:03.485886",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both the ground truth and predicted edit have the type set to \"punctuation\", so the edit type matches exactly.\n\n2. Text Content Accuracy: Both edits identify the same original text (\"high road\") and the same corrected text (\"high-road\"), capturing the identical core change of inserting a hyphen between the words. Line numbers are ignored as instructed, so the difference in line_number does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "tion",
"corrected_text": "tion;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "for what",
"corrected_text": "for what,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "tion,",
"corrected_text": "",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "telling",
"corrected_text": "telling ",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "(comma?)",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:38.867545",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:47:49.813535",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "did",
"corrected_text": "did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "for what",
"corrected_text": "for what,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "tion,",
"corrected_text": "",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "telling",
"corrected_text": "telling",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "said (comma?)",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "n",
"corrected_text": "",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:43.178216",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:33:01.364913",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "tion,",
"corrected_text": "",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "earnestly",
"corrected_text": "earnestly,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:31.173290",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:29:46.817501",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter 1",
"corrected_text": "Chapter 1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:33.591022",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:48:40.664261",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth. Text Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the core intended punctuation change with identical original and corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter 1",
"corrected_text": "Chapter 1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:40.910844",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:33:48.203444",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter 1",
"corrected_text": "Chapter 1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:55.467085",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:30:43.317888",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "late dull autumn",
"corrected_text": "late, dull autumn",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Saone,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:56.138837",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:49:28.638571",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 8,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"man\" in the phrase \"man slowly\" to become \"man, slowly\". The original_text and corrected_text are identical between ground truth and prediction, capturing the intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 8,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change \"Chalons was\" to \"Chalons, was\", inserting a comma after \"Chalons\". This captures the core intended punctuation change exactly. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "late dull",
"corrected_text": "late, dull",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone.",
"corrected_text": "Saone.^ The",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Saone^ it",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:56.794172",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:34:54.998014",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"sodden\" to \"soddened\". The prediction changes \"sodden with\" to \"soddened with\", which includes additional surrounding context (\"with\") but preserves the same core substitution (\"sodden\" \u2192 \"soddened\"). This accurately captures the intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "like a sullied looking-glass",
"corrected_text": "like a sullied looking-glass,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "dull,",
"corrected_text": "",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "river Saone",
"corrected_text": "river Saone,",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:48:54.470713",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:31:44.949312",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "up into his face, with some",
"corrected_text": "up into his face, indeed, with some",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "thanked him from her heart",
"corrected_text": "thanked him with her heart",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "so;",
"corrected_text": ";",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"That's well!",
"corrected_text": "\"That's well! I hoped so;",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:06.864152",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:36:15.666959",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "own",
"corrected_text": "his own",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "up into his face, with some",
"corrected_text": "up into his face, indeed, with some",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "thanked him from her heart",
"corrected_text": "thanked him from her heart,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"Oh! I have very much",
"corrected_text": "\"Oh! I have very much",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "I hoped so;",
"corrected_text": "I hoped so",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "heart",
"corrected_text": "heart,",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "\"That's well! I hoped so;",
"corrected_text": "\"That's well! I hoped so;",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:14.964113",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:32:38.141874",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"hers\" in the phrase \"hers and\", yielding \"hers, and\". Ignoring line numbers as instructed, the predicted edit captures the same intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change\u2014adding a comma after \"hers\". Line number differences are ignored as instructed, so the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"moment with\" to \"moment, with,\" correctly capturing the insertion of a comma after \"moment.\" The core punctuation correction is identical."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"beautiful that\" to \"beautiful, that\", inserting a comma after \"beautiful\". This captures the core intended punctuation change with identical original and corrected text."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation, that papa had abandoned",
"corrected_text": "hesitation, that papa had abandoned,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "he thought directly,",
"corrected_text": "",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:20.693574",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:51:09.608029",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change, inserting a comma after \"hers\" (\"hers and\" \u2192 \"hers, and\"). The original and corrected text strings match the ground truth\u2019s intended correction. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers.\" This captures the core intended correction with identical original and corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original span \"moment with\" and the same correction \"moment, with\", inserting a comma after \"moment\". The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: inserting a comma after \"beautiful\" in the phrase \"beautiful that\" to make it \"beautiful, that\". The original and corrected text strings are identical between ground truth and prediction, capturing the core intended change."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation, that papa had abandoned",
"corrected_text": "hesitation, that papa had abandoned,",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "he thought directly,",
"corrected_text": "",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:19.923880",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:37:14.858094",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"hers and\" to \"hers, and\", inserting a comma after \"hers\". The core punctuation correction is identical in original and corrected text. Line number differences are to be ignored, so this does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction apply the same core change\u2014adding a comma after \"hers\" in the phrase \"hers matters,\" changing it from \"hers matters\" to \"hers, matters.\" The original_text and corrected_text match exactly in both. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both edits change the exact same string from \"moment with\" to \"moment, with,\" inserting a comma after \"moment.\" This captures the core intended punctuation change with identical original and corrected text. Line number differences are irrelevant per instructions."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same corrected text (\"beautiful, that\"), capturing the core intended change of inserting a comma after \"beautiful\". Line numbers are ignored as instructed, so the discrepancy there does not affect correctness."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 31,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change the same word from \"they\" to \"They,\" capturing the identical core capitalization correction. Line number differences are to be ignored, so there is no conflict.\n\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation, that papa had abandoned",
"corrected_text": "hesitation, that papa had abandoned,",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:24.688274",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:33:29.659441",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change of inserting a comma after \"shore\". Line number differences are to be ignored, so the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "flowers pale",
"corrected_text": "flowers, pale",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "within-doors",
"corrected_text": "within-doors",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "within doors",
"corrected_text": "",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:28.881942",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:51:59.451225",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both the ground truth and predicted edit are labeled as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both edits apply the same core punctuation change, inserting a comma after \"shore\" in the phrase \"shore and\" to make it \"shore, and.\" The original_text and corrected_text match the ground truth's intended change.\n\nLine numbers are ignored as instructed, so the difference in line_number does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "once",
"corrected_text": "than",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"Thank you!\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:35.010360",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:38:24.788850",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"shore\" in the phrase \"shore and\" \u2192 \"shore, and\". The predicted edit \"shore\" \u2192 \"shore,\" captures the same core punctuation change (adding a comma immediately after \"shore\"). Although the prediction provides less surrounding context (it omits \"and\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you?",
"corrected_text": "you,\"",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "fancy",
"corrected_text": "fancy,",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "the",
"corrected_text": "",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:55.553941",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:34:30.895293",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 4,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\nText Content Accuracy: The ground truth change is \"city\" \u2192 \"City.\" The predicted edit makes the same core change with identical original and corrected text. Ignoring line numbers as instructed, the prediction captures the intended correction precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "xxix",
"corrected_text": "xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "lord",
"corrected_text": "",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "of",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:50:00.306613",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:53:13.595027",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\" The original_text and corrected_text match the ground truth precisely, capturing the intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 4,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n\nText Content Accuracy: Both identify the same core change, converting \"city\" to \"City\". There is no discrepancy in the original or corrected text content. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 17,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change by inserting a comma immediately after \"existence\" within a slightly larger context (\"existence at\" \u2192 \"existence, at\"). This captures the essential intended punctuation correction.\n\nTherefore, the predicted edit matches both the type and the core textual change of the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 18,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change\u2014adding a comma immediately after \"motionless\"\u2014but includes additional surrounding context (\"motionless when\" \u2192 \"motionless, when\"). This preserves the essential correction intended by the ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 19,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the punctuation following \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The prediction also results in \"stand-still;\" as the corrected form, capturing the core intended outcome (semicolon after \"stand-still\"). Although the predicted original_text omits the comma present in the ground truth original_text, the essential correction (the punctuation after \"stand-still\" should be a semicolon) is correctly represented."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 19,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change by inserting a comma immediately after \"view\" within a larger phrase (\"view by\" \u2192 \"view, by\"). Including the additional context word \"by\" does not alter the essential intended edit, which is the comma after \"view\".\n\nTherefore, both criteria are satisfied."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming.",
"corrected_text": "Dreaming,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "lord",
"corrected_text": "",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "of",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 12,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:36.806380",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:39:50.070890",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of",
"corrected_text": "return of",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "lord",
"corrected_text": "liege",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:55.471547",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:35:27.540664",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts \"of\" to make the phrase \"driest of snorts\" (within the larger context \"of the ... .\"). The predicted edit also replaces \"dryest snorts\" with \"driest of snorts,\" capturing the same core correction (spelling change + insertion of \"of\"). Although the prediction omits surrounding context (\"of the\" and the final period), it still accurately represents the essential intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "been a little",
"corrected_text": "been a little",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "thropic",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a boiling-",
"corrected_text": "",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:49:58.449407",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:54:14.946073",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as a \"replacement,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts an additional \"of\" to make the phrase \"driest of snorts\" (within the larger context \"of the ... .\"). The predicted edit captures the same core transformation: \"dryest snorts\" \u2192 \"driest of snorts.\" Although it omits surrounding context (\"of the\" and the final period), that is acceptable under the criteria since the essential change is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "old boy",
"corrected_text": "old boy;",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "your hands. I authorise",
"corrected_text": "your hands. I authorize",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "thropic",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a",
"corrected_text": "",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:50:01.334978",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:40:55.778181",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 7,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts \"of\" to make the phrase \"the driest of snorts\" (within the larger context \"of the ... .\"). The predicted edit likewise changes \"dryest\" \u2192 \"driest\" and results in \"driest of snorts.\" Although the prediction omits surrounding context (\"of the\" and the final period), it captures the core intended correction (spelling correction plus insertion of \"of\" before \"snorts\"). Therefore, the text content matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 36,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2. Text Content Accuracy: Both edits replace \"authorize\" with \"authorise\", capturing the same core spelling change (American to British spelling). Line numbers are to be ignored, so the mismatch there does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "thropic old buck.",
"corrected_text": "thropic old buck",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "boiling-",
"corrected_text": "",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a",
"corrected_text": "",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:50:00.967324",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-07T19:36:25.382440",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "For",
"corrected_text": "For,",
"observed_line_number": 3,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth intends to insert a comma after the word \"For\" in the phrase \"For Mr. Pancks\" (i.e., \"For\" \u2192 \"For,\"). The predicted edit also changes \"For\" to \"For,\", capturing the same core punctuation insertion, even though it provides less surrounding context. This still matches the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed-",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:50:02.544547",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T00:55:03.805056",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "For",
"corrected_text": "For,",
"observed_line_number": 3,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change inserts a comma after the word \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The prediction captures the same core punctuation change by editing \"For\" \u2192 \"For,\". Although it includes less surrounding context, it correctly represents the essential intended correction: adding a comma immediately after \"For\"."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed-",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_gpt_4_5_preview",
"date": "2026-02-22T15:50:06.897959",
"annotator": "GPT-4.5 Preview (OR)",
"annotation_date": "2025-04-08T07:41:53.554628",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fo",
"corrected_text": "For",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2025-02-27",
"release_source": "https://openai.com/index/introducing-gpt-4-5/",
"release_notes": "OpenRouter route to the same GPT-4.5 Preview release."
},
{
"model_name": "GPT-4o",
"model_id": "gpt-4o",
"date": "2026-02-22T15:26:14.354847",
"shots": 2,
"config": {
"model_id": "gpt-4o",
"display_name": "GPT-4o",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:27:02.706040",
"file_results": [
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:05.679071",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:27:02.706040",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and rattling",
"corrected_text": "pockets, and rattling",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"pockets\" in the phrase \"pockets and\" \u2192 \"pockets, and\". The predicted edit also inserts a comma after \"pockets\" (\"pockets and rattling\" \u2192 \"pockets, and rattling\"). The prediction includes additional following context (\"rattling\"), but the core change\u2014adding the comma after \"pockets\" before \"and\"\u2014is the same. Therefore it captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day, sir",
"corrected_text": "yonder to-day, Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:06.919962",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:19:19.319455",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the core change of inserting a comma after \"yonder\". There are no discrepancies in the intended punctuation change."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Marseilles ;",
"corrected_text": "Marseilles;",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "to-day, sir",
"corrected_text": "to-day, sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:16.377303",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:29:49.885304",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a period at the end of the phrase \"Fellow Travellers\" (i.e., after the word \"Travellers\"). The prediction captures the same core change by editing \"Travellers\" \u2192 \"Travellers.\". Although it includes less surrounding context (omitting \"Fellow\"), it still applies the period at the correct endpoint of the intended text. Therefore, the essential punctuation correction is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same core punctuation change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to become \"pockets, and.\" The original_text and corrected_text match the ground truth\u2019s intended edit. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day, sir",
"corrected_text": "yonder to-day, Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:05.910420",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:27:52.331370",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why did",
"corrected_text": "Why did",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:15.837448",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:20:52.958622",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 27,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify the edit type as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to capitalize the initial \"why\" in the phrase \"why did he dine\" to \"Why did he dine\". The prediction captures the same core change by converting \"why\" to \"Why\". Although it includes less surrounding context, it correctly represents the essential capitalization edit intended in the ground truth."
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:19.433672",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:30:46.378488",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dine to-day,",
"corrected_text": "dine today,",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:24.105911",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:28:39.554606",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 41,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit, so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the word \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam.\" The prediction captures the core intended change by replacing \"said\" with \"asked.\" Although it omits the surrounding context word \"Clennam,\" the essential correction (said \u2192 asked) is correctly represented, and additional context is not required for correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "If",
"corrected_text": "if",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:34.310308",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:21:46.659451",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 38,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase \"if he\" \u2192 \"If he\". The predicted edit capitalizes \"if\" \u2192 \"If\". Although the prediction includes less surrounding context (it does not include the following word \"he\"), it captures the core intended change: capitalizing the word \"if\" at the start of the phrase. Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 41,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the core intended substitution by replacing \"said\" with \"asked\". Although it omits the following context word (\"Clennam\"), this is acceptable because the essential change (said \u2192 asked) is correctly represented."
}
],
"file_id": "005"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:38.437447",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:31:51.453104",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 38,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"if he\" to \"If he\", which is specifically capitalizing the initial \"i\" in \"if\" at the start of the phrase. The prediction changes \"if\" to \"If\", capturing the same core capitalization change, just with less surrounding context (it omits \"he\"). Since the essential intended change\u2014capitalizing \"if\"\u2014is correctly represented, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 40,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes \"said Clennam\" to \"asked Clennam\", whose core intended change is replacing the word \"said\" with \"asked\" while leaving the surrounding context (\"Clennam\") unchanged. The prediction replaces \"said\" with \"asked\" without including the trailing context, but it still captures the essential substitution that the ground truth indicates. Including less context is acceptable as long as the core change is correctly represented."
}
],
"file_id": "005"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:42.366133",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:29:39.014432",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"about and\" \u2192 \"about, and\", inserting a comma after \"about\". The core change is identical with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 28,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction change \"said\" to \"said,\" by adding a comma. This captures the same core correction. Line numbers are to be ignored, so the mismatch there does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 32,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text \"Clennam in\" and the same corrected text \"Clennam, in\", capturing the identical core change (inserting a comma after \"Clennam\").\n\nLine numbers are ignored, so the differing line number does not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "telling him",
"corrected_text": "telling, him",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "highroad",
"corrected_text": "high-road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:38.612886",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:22:41.808497",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit types are \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same correction (\"about, and\"), capturing the identical core change\u2014adding a comma after \"about\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "telling him",
"corrected_text": "telling, him",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "highroad",
"corrected_text": "high-road",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:50.107062",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:32:49.799093",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma so that \"about and\" becomes \"about, and\". This captures the core intended punctuation correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 28,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"said\" to \"said,\" by adding a comma. This captures the same core correction with identical original and corrected text. Line numbers are to be ignored, so the mismatch there does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 32,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change \"Clennam in\" to \"Clennam, in\" by inserting a comma after \"Clennam\". This captures the same core punctuation correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "telling him",
"corrected_text": "telling, him",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "highroad",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:48.616074",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:30:25.011086",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:44.568443",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:23:28.413378",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:54.768997",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:33:40.159305",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:01.704931",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:31:20.529798",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "A late,",
"corrected_text": "A late, dull",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Saone it",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone.",
"corrected_text": "Saone. The",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:12.236858",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:24:32.829058",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are \"replacement\" edits, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes \"sodden\" to \"soddened\". The prediction changes \"sodden with\" to \"soddened with\", which includes extra surrounding context (\"with\") but preserves the same core correction: \"sodden\" \u2192 \"soddened\". Therefore, it correctly captures the intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "A late,",
"corrected_text": "A late, dull",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Chalons lay",
"corrected_text": "Chalons lay a",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone it",
"corrected_text": "Saone it was",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone.",
"corrected_text": "Saone. The",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Chalons was",
"corrected_text": "Chalons was the",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:10.432163",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:34:45.009038",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 8,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma immediately after the word \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). The prediction also inserts a comma after \"Chalons\" (\"Chalons\" \u2192 \"Chalons,\"). Although the prediction includes less surrounding context (it omits the following word \"was\"), it captures the core intended punctuation change at the correct location."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone.",
"corrected_text": "Saone,",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Saone,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:24:55.331938",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:32:14.834997",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "heart,",
"corrected_text": "heart, indeed,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "That's well! I hope so;",
"corrected_text": "That's well! I hoped so;",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:09.028316",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:25:34.509336",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "heart,",
"corrected_text": "heart, indeed,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "well! I",
"corrected_text": "well! I hoped so;",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:13.197771",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:35:52.117037",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The original and corrected text strings match the ground truth\u2019s intended punctuation correction. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "in deed",
"corrected_text": "indeed",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hoped so",
"corrected_text": "hoped so;",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:23.475797",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:33:15.937791",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers.\" This captures the core intended punctuation correction with identical original and corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 21,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change\u2014adding a comma after \"moment\". Ignoring line numbers as instructed, the prediction matches the ground truth intention precisely."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 23,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"beautiful that\" to \"beautiful, that,\" inserting a comma after \"beautiful.\" This captures the core intended punctuation correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "him and trustfulness",
"corrected_text": "him, and trustfulness",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation that",
"corrected_text": "hesitation, that",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this",
"corrected_text": "At this",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly they",
"corrected_text": "directly, they",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:24.554104",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:26:30.854274",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change the phrase from \"hers matters\" to \"hers, matters\", inserting a comma after \"hers\". This captures the same core correction with identical original and corrected text. Line numbers are irrelevant per instructions."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original segment (\"moment with\") and the same correction (\"moment, with\"), correctly capturing the intended insertion of a comma after \"moment\". No discrepancy in the core change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"beautiful that\" to \"beautiful, that,\" inserting a comma after \"beautiful.\" This captures the core intended punctuation change with identical original and corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation that",
"corrected_text": "hesitation, that",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this",
"corrected_text": "At this",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly they",
"corrected_text": "directly, they",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:28.667781",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:37:11.759864",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers\". This captures the same core punctuation correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\n2. Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change: inserting a comma after \"moment\". Any difference in line number is to be ignored, so the text change fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 22,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core change: inserting a comma after \"beautiful\" in the phrase \"beautiful that\" to make \"beautiful, that.\" The original and corrected texts match the ground truth\u2019s intended punctuation change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "him were",
"corrected_text": "him, were",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation that",
"corrected_text": "hesitation, that",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this",
"corrected_text": "At this",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly they",
"corrected_text": "directly, they",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:33.411862",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:34:11.820149",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 23,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same original segment \"shore and\" and the same corrected form \"shore, and,\" capturing the core intended punctuation insertion (a comma after \"shore\"). Line numbers are to be ignored, so there is no mismatch affecting correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction are \"replacement\" edits, so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth replaces \"that\" with \"than\". The prediction replaces the larger phrase \"that once\" with \"than once\", which preserves the core intended change (\"that\" \u2192 \"than\") while including additional surrounding context (\"once\"). This correctly captures the essential correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "whatever Pet was,",
"corrected_text": "whatever Pet was,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that in our fancy",
"corrected_text": "",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\u201cThank you,\u201d",
"corrected_text": "\u201cThank you,\u201d",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:44.830522",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:27:25.172018",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 23,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Ignoring line numbers, the predicted edit matches the ground truth intent."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are of type \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing \"that\" with \"than\". The prediction replaces \"that once\" with \"than once\", which includes additional surrounding context (the word \"once\") but preserves the same core substitution \"that\" \u2192 \"than\". This correctly captures the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that in our fancy",
"corrected_text": "",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Thank you",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:42.191221",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:38:15.143385",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 31,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"that\" to \"than\". The prediction changes \"that once\" to \"than once\", which contains the same core correction (\"that\" \u2192 \"than\") while including additional surrounding context (\"once\"). This still captures the essential intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Pet was,",
"corrected_text": "whatever Pet was,",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now.\u201d",
"corrected_text": "now.\u201d \u201cThank you,\u201d",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "doors when",
"corrected_text": "doors when he entered,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that in our fancy",
"corrected_text": "",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:57.277919",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:35:18.283828",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 3,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n\nText Content Accuracy: Both specify the same core transformation: \"city\" \u2192 \"City\". This captures the essential intended correction in the ground truth. Line numbers are ignored as instructed, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction also inserts a comma immediately after \"existence\" within a slightly larger span (\"existence at\" \u2192 \"existence, at\"). This captures the same core punctuation correction, merely including additional surrounding context. Therefore, the essential edit intention is correctly represented."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 13,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit also inserts a comma immediately after \"motionless\", using additional surrounding context (\"motionless when\" \u2192 \"motionless, when\"). This captures the same core change\u2014adding a comma after \"motionless\"\u2014so the text content matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 14,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth edit inserts a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit also inserts a comma immediately after \"view\" within a longer span (\"view by\" \u2192 \"view, by\"). This preserves the core intended punctuation change (adding a comma after \"view\") while including extra surrounding context, which is allowed.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each",
"corrected_text": "to each",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:26:08.383617",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:28:27.395931",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 3,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The ground truth changes \"city\" to \"City.\" The prediction specifies the same original_text and corrected_text (\"city\" \u2192 \"City\"), capturing the core intended capitalization change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 13,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core punctuation change, adding a comma immediately after \"existence\" in a longer phrase (\"existence at\" \u2192 \"existence, at\"). This preserves the essential intended edit (comma after \"existence\") while including additional surrounding context, which is acceptable."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth\u2019s core change is to have a semicolon after \u201cit\u201d (changing \u201cit,\u201d to \u201cit;\u201d). The predicted edit also results in \u201cit;\u201d (shown as \u201cit; to\u201d with extra right-hand context). Although the predicted \"original_text\" omits the comma shown in the ground truth, the essential intended correction\u2014placing a semicolon immediately after \u201cit\u201d\u2014is captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are labeled as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit changes \"motionless when\" \u2192 \"motionless, when\", which also inserts a comma right after \"motionless\" and includes extra surrounding context (\"when\"). This captures the same core punctuation correction intended by the ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma in the same place within a larger phrase (\"view by\" \u2192 \"view, by\"), which correctly captures the core intended punctuation change. The additional context word \"by\" does not alter the essential edit."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "lord and",
"corrected_text": "lord, and",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:26:04.494862",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:39:23.919776",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 3,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The predicted change is \"city\" \u2192 \"City,\" which matches the ground truth\u2019s core correction exactly (capitalizing the word). Line number differences are ignored as instructed, so they do not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change by inserting a comma immediately after \"existence\" in the broader phrase (\"existence at\" \u2192 \"existence, at\"). This includes additional context but preserves the essential intended punctuation change.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 13,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to add a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same essential change, inserting a comma immediately after \"motionless\" within a slightly larger context (\"motionless when\" \u2192 \"motionless, when\"). This captures the core punctuation correction accurately, and the extra surrounding word does not alter the intent."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 14,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit applies the same core change by inserting a comma after \"view\" within a longer phrase (\"view by\" \u2192 \"view, by\"). The extra context word \"by\" does not alter the essential correction, and the comma placement matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "lord and",
"corrected_text": "lord, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "recurring",
"corrected_text": "to each recurring",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:51.388444",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:36:13.303945",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "of the dryest snorts",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:53.003696",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:29:22.537527",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "snorts",
"corrected_text": "snorts.",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest",
"corrected_text": "of the dryest",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:25:59.868780",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:40:25.210025",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 36,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The ground truth edit is a \"replacement\" and the predicted edit is also a \"replacement\", so the edit type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction replace the same original text \"authorize\" with the same corrected text \"authorise\". This captures the core intended change (American to British spelling). Line numbers are ignored as instructed, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts.",
"corrected_text": "of snorts.",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:26:01.853842",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T19:37:08.154939",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks.\" The prediction also inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the core intended punctuation change at the correct location. Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:26:11.173639",
"annotator": "GPT-4o",
"annotation_date": "2025-04-07T23:30:21.821833",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "For,",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-4o",
"date": "2026-02-22T15:26:14.354847",
"annotator": "GPT-4o",
"annotation_date": "2025-04-08T08:41:20.497225",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction also inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the predicted span is shorter and omits \"Pancks,\" it captures the same core punctuation change in the correct location. Therefore, the essential edit is accurately represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2024-05-13",
"release_source": "https://openai.com/index/hello-gpt-4o/"
},
{
"model_name": "GPT-4o (2024-11-20 OR)",
"model_id": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T16:06:21.434675",
"shots": 2,
"config": {
"model_id": "or_gpt_4o_2024_11_20",
"display_name": "GPT-4o (2024-11-20 OR)",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:06:16.341168",
"file_results": [
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:13.281744",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:06:16.341168",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth adds a period at the end of the phrase \"Fellow Travellers\" (making it \"Fellow Travellers.\"). The predicted edit adds a period at the end of the final word \"Travellers\" (making it \"Travellers.\"). Although the prediction includes less surrounding context (it omits \"Fellow\"), it captures the core intended change: inserting a period immediately after \"Travellers\" at the end of the phrase. Therefore, the essential punctuation correction is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edit have the type \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both specify the same original text \"yonder to-day\" and the same correction \"yonder, to-day\", inserting a comma after \"yonder\". The core change is identical, so the intended edit is fully captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "pockets and rattling",
"corrected_text": "pockets, and rattling",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"pockets\" in the phrase \"pockets and\" \u2192 \"pockets, and.\" The predicted edit makes the same core punctuation change (\"pockets and rattling\" \u2192 \"pockets, and rattling\"), simply including additional surrounding context. The essential intended correction\u2014adding a comma after \"pockets\" before \"and\"\u2014is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day, sir",
"corrected_text": "to-day, Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:17.901838",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:01:48.133301",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates adding a period at the end of the phrase \"Fellow Travellers\" (i.e., \"Fellow Travellers\" \u2192 \"Fellow Travellers.\"). The prediction applies the same core change\u2014adding a period at the end of the final word \"Travellers\" (\"Travellers\" \u2192 \"Travellers.\"). Although the predicted edit includes less surrounding context, it still captures the essential intended correction: inserting a period after the word \"Travellers\" at the end of the heading/phrase."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day, sir",
"corrected_text": "yonder, to-day, sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"yonder\" (\"yonder to-day\" \u2192 \"yonder, to-day\"). The predicted edit also inserts a comma after \"yonder\" within a longer phrase (\"yonder to-day, sir\" \u2192 \"yonder, to-day, sir\"). The additional context (\", sir\") does not alter the core intended correction, and the essential punctuation change is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "pockets and rattling",
"corrected_text": "pockets, and rattling",
"observed_line_number": 15,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"pockets\" (\"pockets and\" \u2192 \"pockets, and\"). The prediction applies the same core punctuation change but includes extra surrounding context (\"pockets and rattling\" \u2192 \"pockets, and rattling\"). This correctly captures the essential intended edit: adding a comma after \"pockets.\""
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:09.148758",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:43:47.297679",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both edits change \"yonder to-day\" to \"yonder, to-day\", inserting a comma after \"yonder\". The core punctuation change is identical in the prediction and ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "pockets and rattling",
"corrected_text": "pockets, and rattling",
"observed_line_number": 14,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"pockets\" in the phrase \"pockets and\" \u2192 \"pockets, and.\" The prediction applies the same core punctuation change within a longer context: \"pockets and rattling\" \u2192 \"pockets, and rattling.\" The essential edit (adding a comma after \"pockets\" before \"and\") is correctly captured, and the extra surrounding words are acceptable context."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day, sir",
"corrected_text": "to-day, Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Marseilles people",
"corrected_text": "Marseilles' people",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shutting 'em up",
"corrected_text": "shutting 'em, up",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:08.089180",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:07:10.423917",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones\"",
"corrected_text": "stones.\"",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:14.604181",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:02:40.377516",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 26,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so this matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the phrase \"why did he dine\" to \"Why did he dine\", which is specifically capitalizing the initial \"w\" in \"why\". The prediction captures this core change by editing \"why\" \u2192 \"Why\". Although it includes less surrounding context than the ground truth, it correctly represents the essential capitalization correction intended."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones\" She",
"corrected_text": "stones.\" She",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:08.820530",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:44:34.393164",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones\" She",
"corrected_text": "stones.\" She",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:22.037274",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:07:56.974106",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 38,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the phrase \"if he\" to \"If he\", which capitalizes the word \"if\" at the start of the phrase. The prediction changes \"if\" to \"If\". Although the prediction includes less surrounding context (it omits \"he\"), it captures the core intended change: lowercasing \"if\" to capitalized \"If\". Additional context is not required as long as the essential change is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 40,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"said Clennam\" to \"asked Clennam\", whose essential correction is replacing the word \"said\" with \"asked\" while keeping the surrounding context (\"Clennam\") unchanged. The prediction replaces \"said\" with \"asked\" (without including \"Clennam\"), which still captures the core intended change. Including less surrounding context is acceptable as long as the essential substitution is correct."
}
],
"file_id": "005"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:27.513043",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:45:23.293987",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:33.677406",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:08:49.430754",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"high road\") and the same corrected text (\"high-road\"), capturing the identical core change of adding a hyphen. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core correction: inserting a comma so that \"about and\" becomes \"about, and.\" The original_text and corrected_text match the ground truth\u2019s intended change. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 28,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change the word \"said\" to \"said,\" by adding a comma. This captures the same core punctuation correction. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:51.606938",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:04:22.407265",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 12,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma so that \"about and\" becomes \"about, and\". The original and corrected text match the ground truth intention precisely. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both edits change \"said\" to \"said,\", inserting a comma after the word. This captures the core intended correction with identical original and corrected text. Line number differences are to be ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 30,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Clennam in\" to \"Clennam, in\", inserting a comma after \"Clennam\". This captures the same core punctuation correction. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "highroad",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:41.713617",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:46:20.581504",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road,\" capturing the identical core change (inserting a hyphen between the words). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma between \"about\" and \"and\" (\"about and\" \u2192 \"about, and\"). The original and corrected text match exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2. Text Content Accuracy: Both specify the same core correction: changing \"said\" to \"said,\" by adding a comma. This captures the essential intended edit with no discrepancy in the text change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 31,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core correction: inserting a comma after \"Clennam\" in the phrase \"Clennam in\" to make \"Clennam, in.\" The original and corrected texts match the ground truth\u2019s intended change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:42.506537",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:09:53.887868",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:50.451682",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:05:08.380799",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:49.135290",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:47:13.439749",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to make the phrase end with a period (\"Sun and Shadow\" \u2192 \"Sun and Shadow.\"). The prediction also results in the same intended correction (ending with a period: \"Sun and Shadow.\"), even though it assumes the original had a trailing comma (\"Sun and Shadow,\"). The core editorial intention\u2014ensuring the text ends with a period\u2014is correctly captured."
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:03.250123",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:10:40.780501",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Ch\u00e2lons",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:52:58.679631",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:06:13.779136",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Chalons",
"corrected_text": "Ch\u00e2lons",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "soddenwith",
"corrected_text": "sodden with",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:03.565188",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:48:03.433697",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Chalons",
"corrected_text": "Ch\u00e2lons",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "soddenwith",
"corrected_text": "sodden with",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:07.891316",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:11:38.839721",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"heart so\" to \"heart, so\", inserting a comma after \"heart\". The core change is identical, with no discrepancies in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hoped so",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:15.442630",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:07:09.372548",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same core change: inserting a comma after \"heart\" in the phrase \"heart so\" to become \"heart, so.\" The original_text and corrected_text match the ground truth\u2019s intended punctuation edit. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hoped so",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:17.454940",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:49:06.429445",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core change: inserting a comma after \"heart\" in the phrase \"heart so\" to make it \"heart, so.\" The original and corrected text match the ground truth\u2019s intention. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "face with",
"corrected_text": "face, with",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "her heart",
"corrected_text": "her heart,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Oh I",
"corrected_text": "Oh! I",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hoped so",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:21.871952",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:12:43.897381",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "on hers matters",
"corrected_text": "on hers, matters",
"observed_line_number": 13,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The prediction applies the same core punctuation change within a slightly larger context (\"on hers matters\" \u2192 \"on hers, matters\"), which correctly captures the essential intended correction. Extra surrounding words do not alter the core edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "for a moment with",
"corrected_text": "for a moment, with",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth inserts a comma after \"moment\" (\"moment with\" \u2192 \"moment, with\"). The prediction applies the same core punctuation change, inserting a comma after \"moment\" as well, but includes extra surrounding context (\"for a moment with\" \u2192 \"for a moment, with\"). This additional context does not change the essential edit, and the intended punctuation correction is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation that",
"corrected_text": "hesitation, that",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this",
"corrected_text": "At this",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:23.270237",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:07:58.434796",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "on hers matters",
"corrected_text": "on hers, matters",
"observed_line_number": 13,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma between \"hers\" and \"matters\" (\"hers matters\" \u2192 \"hers, matters\"). The prediction performs the same essential punctuation change, adding a comma after \"hers\" (\"on hers matters\" \u2192 \"on hers, matters\"). The extra surrounding context word (\"on\") does not alter the core intended edit. Therefore, the text content change matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "for a moment with",
"corrected_text": "for a moment, with",
"observed_line_number": 20,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"moment\" (\"moment with\" \u2192 \"moment, with\"). The predicted edit performs the same core change, also inserting a comma after \"moment\" (\"for a moment with\" \u2192 \"for a moment, with\"). The prediction includes additional surrounding context (\"for a\") but preserves the essential intended punctuation change.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation that",
"corrected_text": "hesitation, that",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this",
"corrected_text": "At this",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:25.733384",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:49:55.566197",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "on hers matters",
"corrected_text": "on hers, matters",
"observed_line_number": 14,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" in the phrase \"hers matters\" \u2192 \"hers, matters\". The predicted edit applies the same essential punctuation change within a longer snippet: \"on hers matters\" \u2192 \"on hers, matters\". The additional context word (\"on\") does not alter the core intended correction, which is the comma insertion after \"hers\".\n\nTherefore, the prediction correctly captures the ground truth edit\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "for a moment with",
"corrected_text": "for a moment, with",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"moment\" in the phrase \"moment with\" \u2192 \"moment, with\". The prediction does the same core change, adding a comma after \"moment\" in a slightly longer surrounding context: \"for a moment with\" \u2192 \"for a moment, with\". This captures the same intended punctuation correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation that",
"corrected_text": "hesitation, that",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this",
"corrected_text": "At this",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:41.597743",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:13:36.657097",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 22,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\" before \"and\". Any difference in line number is to be ignored per instructions.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 32,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the word \"that\" to \"than\". The predicted edit changes \"that once\" to \"than once\", which includes additional surrounding context (the word \"once\") but preserves the same core correction: \"that\" \u2192 \"than\". This accurately captures the ground truth\u2019s intended change."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that in our fancy",
"corrected_text": "in our fancy",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "\u201cThank you\u201d",
"corrected_text": "\u201cThank you,\u201d",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:43.169102",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:08:48.557746",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 22,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original_text (\"shore and\") and the same corrected_text (\"shore, and\"), capturing the identical core change (inserting a comma after \"shore\"). Ignoring line numbers as instructed, the prediction matches the ground truth edit intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 31,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction use the edit type \"replacement\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is specifically \"that\" \u2192 \"than\". The prediction expands the span to \"that once\" \u2192 \"than once\", but the essential intended correction (changing \"that\" to \"than\") is preserved, with \"once\" simply included as additional unchanged context. Therefore the core textual correction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that in our fancy",
"corrected_text": "in our fancy",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "whatever Pet was,",
"corrected_text": "whatever Pet was the other was,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "\u201cThank you,\u201d",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "The lights were bright",
"corrected_text": "The lights were bright within-doors",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:43.261970",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:50:47.035506",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 22,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Ignoring line numbers, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 25,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n\nText Content Accuracy: Both specify replacing \"within doors\" with \"within-doors\", capturing the same core intended correction (hyphenation). No discrepancy in the actual text change. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that in our fancy",
"corrected_text": "",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "Thank you,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:54:18.462626",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:14:37.392621",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 3,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n\nText Content Accuracy: Both specify the same core change: \"city\" is corrected to \"City\". This captures the essential intended correction with identical original and corrected text.\n\nLine numbers are ignored as instructed, so the differing line_number values do not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to make the punctuation after \u201cthem\u201d a semicolon (i.e., \u201cthem,\u201d \u2192 \u201cthem;\u201d). The prediction also results in \u201cthem;\u201d. Although the prediction\u2019s `original_text` omits the comma (\"them\" instead of \"them,\"), it still captures the core intended correction: the punctuation following \u201cthem\u201d should be a semicolon. Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates inserting a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change, inserting a comma immediately after \"existence\" while including extra surrounding context (\"existence at\" \u2192 \"existence, at\"). This captures the essential intended punctuation correction.\n\nTherefore, the prediction correctly reflects the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 19,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma immediately after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction also inserts a comma after \"view,\" using additional context (\"view by\" \u2192 \"view, by\"). This captures the same core punctuation change at the same location relative to the word \"view.\" Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "when we",
"corrected_text": "when, we",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "lord",
"corrected_text": "Lord",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T16:06:21.434675",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:09:44.530271",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 3,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"city\" to \"City,\" capturing the same core capitalization correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth edit inserts a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit applies the same core change by inserting a comma immediately after \"existence\" within a larger phrase (\"existence at\" \u2192 \"existence, at\"). The additional context word \"at\" does not alter the essential intention of the edit. Therefore, the predicted edit correctly captures the ground truth change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 17,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit changes \"motionless when\" \u2192 \"motionless, when\", which also inserts a comma immediately after \"motionless\" and simply includes the following word as additional context. This captures the same core punctuation change.\n\nTherefore, the predicted edit correctly matches the intention of the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 18,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the types match exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma immediately after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction inserts a comma after \"view\" as well, using additional surrounding context (\"view by\" \u2192 \"view, by\"). This captures the same core punctuation change intended by the ground truth.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "lord",
"corrected_text": "Lord",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:54:05.808046",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:51:54.964677",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"city\" is corrected to \"City\". There is no discrepancy in the original or corrected text content. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view;",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "lord",
"corrected_text": "liege lord",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each",
"corrected_text": "to each",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:50.899963",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:15:37.336020",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"authorize\" with \"authorise\", which captures the same core spelling change intended by the ground truth. Ignoring line numbers, the essential edit is identical."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:54:00.476439",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:10:42.425995",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 36,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"authorize\" with \"authorise\", capturing the same core spelling change. Ignoring line numbers as instructed, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:53:58.457141",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:52:52.513274",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify replacing \"authorize\" with \"authorise\", capturing the same core spelling change. Line number differences are ignored as instructed. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "of the dryest snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:54:00.768404",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T00:16:23.508602",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:54:00.767062",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T01:11:28.398685",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "making me",
"corrected_text": "making me,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_gpt_4o_2024_11_20",
"date": "2026-02-22T15:54:06.731205",
"annotator": "GPT-4o (2024-11-20 OR)",
"annotation_date": "2025-04-08T08:53:42.363017",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "still 'tis",
"corrected_text": "still, 'tis",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2024-11-20",
"release_source": "https://help.openai.com/en/articles/9624314-model-release-notes"
},
{
"model_name": "GPT-4o (OR)",
"model_id": "or_gpt_4_turbo",
"date": "2026-02-22T15:52:02.560089",
"shots": 2,
"config": {
"model_id": "or_gpt_4_turbo",
"display_name": "GPT-4o (OR)",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:20:57.604638",
"file_results": [
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:49:58.279031",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:20:57.604638",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to-day, Sir,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling,over",
"corrected_text": "howling, over",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:10.787125",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:43:19.234870",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": ",",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to-day, Sir,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:10.786476",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:11:53.967140",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to-day, Sir,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling,over",
"corrected_text": "howling, over",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:19.267251",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:21:49.476179",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "misgiving",
"corrected_text": "reason",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:19.269710",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:44:07.356145",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine",
"corrected_text": "why did he dine to",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "CAP",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:24.303933",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:12:47.237409",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine",
"corrected_text": "why did he dine to",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:24.306439",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:22:37.079597",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:29.845727",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:45:06.735455",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:29.724874",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:13:39.037019",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:39.815366",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:23:33.002073",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"said when\" to \"said, when\", which is effectively the insertion of a comma immediately after \"said\". The prediction captures this core change by converting \"said\" to \"said,\". Although it omits the following word \"when\" from the span, it still represents the same essential punctuation insertion at the correct location. Therefore, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:36.823859",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:46:04.911365",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 27,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same core correction\u2014adding a comma after \"said\" (\"said\" \u2192 \"said,\"). Ignoring line numbers as instructed, the predicted edit captures the same essential punctuation change as the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:38.941518",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:24:18.282116",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the same core correction with identical original and corrected text. Line number differences are ignored as instructed."
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:40.070473",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:46:53.951141",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both edits change the same phrase from \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the core intended correction with no discrepancy in the actual text change.\n\nLine numbers are ignored as instructed, so the difference in line_number does not affect correctness."
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:39.570272",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:15:15.971735",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the same core punctuation correction with identical original and corrected text. Line numbers are ignored as instructed."
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:50.910635",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:25:06.155274",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "looking-glass in",
"corrected_text": "looking-glass, in",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "river Saone it",
"corrected_text": "river Saone, it",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with",
"corrected_text": "sodden, with",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:52.101977",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:47:53.594685",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "looking-glass in",
"corrected_text": "looking-glass, in",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone it",
"corrected_text": "Saone, it",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with",
"corrected_text": "sodden, with",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:55.109069",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:16:04.244118",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "looking-glass in",
"corrected_text": "looking-glass, in",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone it",
"corrected_text": "Saone, it",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with",
"corrected_text": "sodden, with",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:50:52.841360",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:26:01.285472",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I would have",
"corrected_text": "I wouldhave",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hoped",
"corrected_text": "hoped so",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hope so",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:09.677962",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:48:52.066063",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I would have",
"corrected_text": "I wouldhave",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hoped",
"corrected_text": "hope",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hope so",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:06.778604",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:17:09.590580",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I would have",
"corrected_text": "I wouldhave",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hoped",
"corrected_text": "hoped so",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "generous",
"corrected_text": "",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hoped so",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:04.534623",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:26:51.111620",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "hers",
"corrected_text": "Hers",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:03.554963",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:49:46.511089",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:09.182302",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:18:00.936677",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "cap",
"corrected_text": "Cap",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hers",
"corrected_text": "his",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:25.754588",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:27:47.478556",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "that once were",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:23.469936",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:50:43.748454",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "that once were",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:28.725471",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:18:55.972801",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "that once were",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:43.834558",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:28:44.308180",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 18,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core punctuation change, inserting a comma immediately after \"view\" within a larger context (\"view by\" \u2192 \"view, by\"). This captures the essential intended edit (comma after \"view\"), with permissible extra surrounding words."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city (CCCP)",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "lucrative",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence; at",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:41.999437",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:51:38.707310",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city (CCCP)",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "lucrative",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "liege",
"corrected_text": "lord",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:45.233741",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:19:54.337922",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city (CCQ)",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "lucrative",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "liege",
"corrected_text": "liege lord",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:33.307448",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:29:46.184630",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "driest of snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "boiling-over",
"corrected_text": "boiling over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:34.205065",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:52:30.118701",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "driest of snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "boiling-over",
"corrected_text": "boiling over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:51:44.913950",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T21:30:32.295312",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "so quickly that",
"corrected_text": "so quickly, that",
"observed_line_number": 9,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"quickly\" (\"quickly that\" \u2192 \"quickly, that\"). The prediction also inserts a comma after \"quickly\" in the same construction, just with extra preceding context (\"so quickly that\" \u2192 \"so quickly, that\"). This captures the same core punctuation correction intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "making me",
"corrected_text": "making me,",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I ought",
"corrected_text": "I ought,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur",
"corrected_text": "Arthur,",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "still 'tis",
"corrected_text": "still, 'tis",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and if",
"corrected_text": "and, if",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I did",
"corrected_text": "I did,",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I suppose",
"corrected_text": "I suppose,",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "but I'm",
"corrected_text": "but, I'm",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "and a",
"corrected_text": "",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:52:02.560089",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-07T23:53:22.654194",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sight",
"corrected_text": "sight,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "mind",
"corrected_text": "mind,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "apology",
"corrected_text": "apology,",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Arthur",
"corrected_text": "Arthur,",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me",
"corrected_text": "me,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_gpt_4_turbo",
"date": "2026-02-22T15:52:02.036659",
"annotator": "GPT-4o (OR)",
"annotation_date": "2025-04-08T06:21:34.249513",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 9,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"quickly that\" \u2192 \"quickly, that\", inserting a comma after \"quickly\". This captures the core intended punctuation change with identical original and corrected text. Line number differences are irrelevant as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sight as this and",
"corrected_text": "sight as this, and",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "making me",
"corrected_text": "making me,",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur for",
"corrected_text": "Arthur, for",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Arthur returned",
"corrected_text": "Arthur, returned",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view but",
"corrected_text": "view, but",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "a",
"corrected_text": "",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2023-11-06",
"release_source": "https://openai.com/index/new-models-and-developer-products-announced-at-devday/",
"release_notes": "OpenRouter route to GPT-4 Turbo."
},
{
"model_name": "GPT-4o Mini",
"model_id": "gpt-4o-mini",
"date": "2026-02-22T15:28:07.409022",
"shots": 2,
"config": {
"model_id": "gpt-4o-mini",
"display_name": "GPT-4o Mini",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:39:04.893095",
"file_results": [
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:23.716969",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:39:04.893095",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "is none.",
"corrected_text": "is none.",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "other-wise.",
"corrected_text": "otherwise.",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Marseilles,",
"corrected_text": "Marseilles.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "business,",
"corrected_text": "business.",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day.",
"corrected_text": "to-day.",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day!",
"corrected_text": "to-day!",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "in for?",
"corrected_text": "in for?",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day, Sir,",
"corrected_text": "yonder to-day, sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:24.022049",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:33:12.387968",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to be heard.",
"corrected_text": "to be heard.",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "happy other-wise.",
"corrected_text": "happy otherwise.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "insurrectionary tune into",
"corrected_text": "insurrectionary tune into",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "something or other \u2014",
"corrected_text": "something or other\u2014",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "business, instead of shutting 'em in quarantine!",
"corrected_text": "business instead of shutting 'em in quarantine!",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day.",
"corrected_text": "to-day.",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day! What have we ever been in for?",
"corrected_text": "to-day! What have we ever been in for?",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day, Sir,",
"corrected_text": "yonder to-day, sir,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:30.550475",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:16:39.557325",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day.",
"corrected_text": "to-day.",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day!",
"corrected_text": "to-day!",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day.",
"corrected_text": "to-day.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day.",
"corrected_text": "to-day.",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day, Sir,",
"corrected_text": "yonder to-day, sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:37.749358",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:40:01.422174",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "the great difference that",
"corrected_text": "the great difference between",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and he gentle",
"corrected_text": "and he, gentle",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I knew you must",
"corrected_text": "I knew you must be",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and is so cold.",
"corrected_text": "and, it is so cold.",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to touch it, while",
"corrected_text": "to touch it while",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "put it nearer the warmth.",
"corrected_text": "put it nearer, the warmth.",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it was quite warm;",
"corrected_text": "it was quite warm.",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to people.",
"corrected_text": "to people?",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I tell you something, sir?",
"corrected_text": "I tell you something, sir.",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to the mercy of the cold stones.",
"corrected_text": "to the mercy of the cold stones?",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:30.788419",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:34:01.330640",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the great difference that",
"corrected_text": "the great difference that,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and is so cold.",
"corrected_text": "and is so cold;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the fire; \"put it",
"corrected_text": "the fire; \"put it,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the cold stones\"",
"corrected_text": "the cold stones\";",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to the mercy of the cold stones\"",
"corrected_text": "to the mercy of the cold stones\";",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"may I tell you something, sir?\"",
"corrected_text": "\"may I tell you something, sir?\"",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:52.178223",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:17:36.280926",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "my poor child! Here at midnight?",
"corrected_text": "my poor child! Here at midnight?",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you must be very much surprised.",
"corrected_text": "you must be very much surprised.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I have no fire, said Clennam.",
"corrected_text": "I have no fire,\u201d said Clennam.",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it is so cold.",
"corrected_text": "it is so cold,\u201d",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "put it nearer the warmth.",
"corrected_text": "put it nearer the warmth,\u201d",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it was quite warm!",
"corrected_text": "it was quite warm!\u201d",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I tell you something, sir?",
"corrected_text": "I tell you something, sir?\u201d",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the cold stones!",
"corrected_text": "the cold stones?\u201d",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:34.723943",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:41:03.859936",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:34.834445",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:34:52.868577",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:43.053654",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:18:26.847524",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:52.476283",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:42:03.583185",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "said (comma)",
"corrected_text": "said,",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \u201csaid\u201d in the phrase \u201csaid when\u201d \u2192 \u201csaid, when.\u201d The prediction also captures the essential change by adding a comma after \u201csaid\u201d (\u201csaid\u201d \u2192 \u201csaid,\u201d). Although it does not include the following word \u201cwhen,\u201d it still represents the same core punctuation correction (comma after \u201csaid\u201d). Therefore, the prediction matches the ground truth\u2019s intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:55.485316",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:35:41.682468",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said (comma)",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"said\" in the sequence \"said when\" \u2192 \"said, when.\" The predicted edit also represents inserting a comma immediately after \"said\" (\"said\" \u2192 \"said,\"). Although the prediction omits the following word \"when\" and uses a slightly odd original_text representation (\"said (comma)\"), it still captures the core intended change: adding a comma after \"said.\" Therefore, the essential punctuation correction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam!",
"corrected_text": "Clennam,",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam!",
"corrected_text": "Clennam,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:58.353239",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:19:28.668572",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "said (comma)",
"corrected_text": "said,",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"said\" in the phrase \"said when\" \u2192 \"said, when\". The predicted edit represents the same core punctuation change by converting \"said\" into \"said,\" (even though it does not include the following word \"when\" and oddly includes \"(comma)\" in the original_text). The essential intended modification\u2014adding a comma after \"said\"\u2014is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:26:58.265772",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:42:47.030053",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:02.822380",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:36:29.142886",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:01.707482",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:20:11.530035",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to make the title end with a period (\"Sun and Shadow\" \u2192 \"Sun and Shadow.\"). The predicted edit also results in the same corrected text ending with a period (\"Sun and Shadow,\" \u2192 \"Sun and Shadow.\"). Although the prediction assumes the original ended with a comma rather than no punctuation, it still captures the core intended correction: the ending punctuation should be a period."
}
],
"file_id": "007"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:22.927557",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:43:45.052827",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Let Loose ,",
"corrected_text": "Let Loose.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to add a period to make the phrase read \"Let Loose.\" The prediction also results in \"Let Loose.\" While the predicted original_text includes an extra comma (\"Let Loose ,\") that is not present in the ground truth original_text, the essential intended correction\u2014ensuring the phrase ends with a period\u2014is correctly captured in the predicted corrected_text. Therefore, the core change matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI.",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the wa-",
"corrected_text": "the hyphen",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sa\u00f4ne. The",
"corrected_text": "Sa\u00f4ne. the",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with wet;",
"corrected_text": "soddened with wet;",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:17.404421",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:37:19.247316",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hyphen",
"corrected_text": "hyphen.",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sodden with wet",
"corrected_text": "soddened with wet",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "toil and muttering.",
"corrected_text": "toiling and muttering.",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sa\u00f4ne. The",
"corrected_text": "Sa\u00f4ne, the",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:29.114617",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:21:01.762011",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing \"sodden\" with \"soddened.\" The prediction replaces \"sodden with\" with \"soddened with,\" which includes extra surrounding context (the word \"with\") but preserves the same core modification: \"sodden\" \u2192 \"soddened.\" Therefore, the predicted edit captures the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sa\u00f4ne. The",
"corrected_text": "Sa\u00f4ne, the",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "hyphen",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:13.897925",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:44:42.522930",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hoped so",
"corrected_text": "I hope so",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cOh! I have very much to ask of you.\u201d",
"corrected_text": "\u201cOh! I have very much to ask of you?\u201d",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:24.825406",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:38:12.746722",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "him",
"corrected_text": "himself",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cOh! I have very much to ask of you.\u201d",
"corrected_text": "\u201cOh! I have very much to ask of you?\u201d",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\u201cThat\u2019s well! I hoped so; I am not disappointed.\u201d",
"corrected_text": "\u201cThat\u2019s well! I hoped so; I am not disappointed!\u201d",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "I hoped so",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:14.176972",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:22:06.349541",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hoped so",
"corrected_text": "I hoped so;",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I have very much to ask of you.",
"corrected_text": "I have very much to ask of you?",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Oh!",
"corrected_text": "Oh!",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to his face, with some hurried words",
"corrected_text": "to his face; with some hurried words",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:20.998399",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:45:32.675469",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "rose or two from hers",
"corrected_text": "rose or two from her",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and out at that arch of light at the other end,",
"corrected_text": "and out at that arch of light, at the other end,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that papa had abandoned the idea.",
"corrected_text": "that papa had abandoned the idea.",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this, he thought directly,",
"corrected_text": "At this, he thought directly,",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to be married.",
"corrected_text": "to be married?",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "for a moment, with a look in which regard for him and trustfulness in him were strikingly blended",
"corrected_text": "for a moment, with a look in which regard for him and trustfulness in him, were strikingly blended",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:28.642651",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:38:58.717952",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "rose or two from hers",
"corrected_text": "rose or two from her's",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and out at that arch of light at the other end",
"corrected_text": "and out at that arch of light; at the other end",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "for a moment, with a look",
"corrected_text": "for a moment; with a look",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that papa had abandoned",
"corrected_text": "that papa had abandoned.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this, he thought directly,",
"corrected_text": "At this, he thought directly,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:29.548716",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:22:59.311902",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "rose or two from hers",
"corrected_text": "rose or two from her's",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and out at that arch of light at the other end",
"corrected_text": "and out at that arch of light; at the other end",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "for a moment, with a look",
"corrected_text": "for a moment; with a look",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that papa had abandoned",
"corrected_text": "that papa had abandoned.",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "at this, he thought directly,",
"corrected_text": "At this, he thought directly,",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:45.540219",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:46:21.917414",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once were",
"corrected_text": "than once were",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in our fancy",
"corrected_text": "that in our fancy,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors.",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:42.157451",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:39:44.902906",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors.",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once were",
"corrected_text": "than once were",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in our fancy",
"corrected_text": "that in our fancy,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:52.863329",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:23:57.556642",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 27,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"within doors\" with \"within-doors\". The core intended change (hyphenating the phrase) is identical. Line number differences are to be ignored, so the prediction captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in our fancy",
"corrected_text": "that in our fancy,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "whatever Pet was, the other was?",
"corrected_text": "whatever Pet was, the other was?",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "than",
"corrected_text": "than,",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:59.580088",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:47:17.778070",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the",
"corrected_text": "The",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "each",
"corrected_text": "each recurring",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "being has.",
"corrected_text": "being has;",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "when we were brought",
"corrected_text": "when we were brought",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege lord",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence; at",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to",
"corrected_text": "it; to",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view; by",
"corrected_text": "view; by",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; the",
"corrected_text": "existence; the",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:58.879910",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:40:45.261079",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the",
"corrected_text": "The",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to each",
"corrected_text": "each",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "when we were",
"corrected_text": "when we were",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view; by",
"corrected_text": "view; by",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; the",
"corrected_text": "existence; the",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "liege lord",
"corrected_text": "liege lord.",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence; at",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it; to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:28:00.474335",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:24:55.152798",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the",
"corrected_text": "The",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them,",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it;",
"corrected_text": "it,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "when",
"corrected_text": "when,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view;",
"corrected_text": "view,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege",
"corrected_text": "liege,",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "city",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence;",
"corrected_text": "existence,",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:49.740117",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:48:09.330165",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of the driest snorts.",
"corrected_text": "of the driest snorts.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "your hands.",
"corrected_text": "your hands.",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:49.736523",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:41:43.428260",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of the driest snorts.",
"corrected_text": "of the driest snorts.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bubbling out of him",
"corrected_text": "bubbling out of him",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "your hands.",
"corrected_text": "your hands.",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:27:57.183223",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:25:45.166225",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of the driest snorts.",
"corrected_text": "of the driest snorts.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bubbling out of him",
"corrected_text": "bubbling out of him",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "your hands.",
"corrected_text": "your hands.",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:28:05.684245",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T19:49:05.246399",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Rugg,",
"corrected_text": "Rugg;",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "never and a faded check",
"corrected_text": "never, and a faded check",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me, not that",
"corrected_text": "me; not that",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fled Arthur,",
"corrected_text": "fled, Arthur,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,",
"corrected_text": "view;",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "head.",
"corrected_text": "head;",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:28:05.682601",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-07T21:42:35.520713",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Rugg,",
"corrected_text": "Rugg.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me, not",
"corrected_text": "me; not",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fled Arthur,",
"corrected_text": "fled, Arthur,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Cucumber",
"corrected_text": "Cucumber,",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-4o-mini",
"date": "2026-02-22T15:28:07.409022",
"annotator": "GPT-4o Mini",
"annotation_date": "2025-04-08T08:26:45.130120",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Rugg,",
"corrected_text": "Rugg;",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "never and a faded check",
"corrected_text": "never, and a faded check",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me, not that",
"corrected_text": "me; not that",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fled Arthur,",
"corrected_text": "fled, Arthur,",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,",
"corrected_text": "view;",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "head.",
"corrected_text": "head;",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2024-07-18",
"release_source": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/"
},
{
"model_name": "GPT-4o-mini (OR)",
"model_id": "or_gpt_4o_mini",
"date": "2026-02-22T15:56:02.218007",
"shots": 2,
"config": {
"model_id": "or_gpt_4o_mini",
"display_name": "GPT-4o-mini (OR)",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-08",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:20:10.302989",
"file_results": [
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:19.636200",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:20:10.302989",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I mean the French people.",
"corrected_text": "I mean the French people; they're always at it.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the world that was ever composed.",
"corrected_text": "the world that was ever composed; it couldn't exist without",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to something or other \u2014 victory or death, or blazes,",
"corrected_text": "to something or other \u2014 victory or death, or blazes; or something.",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shutting 'em up in quarantine!",
"corrected_text": "shutting 'em up in quarantine.",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Out to-day!",
"corrected_text": "Out to-day!",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "What have we ever been in for?",
"corrected_text": "What have we ever been in for?",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day, Sir,",
"corrected_text": "yonder to-day, sir,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:15.002427",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:45:18.660436",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to-day",
"corrected_text": "to-day,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to-day.",
"corrected_text": "to-day!",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to-day.",
"corrected_text": "to-day!",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to-day.",
"corrected_text": "to-day!",
"observed_line_number": 43,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "and marshong",
"corrected_text": "and marshong,",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:15.144204",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-08T06:31:18.703268",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to-day",
"corrected_text": "to-day,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to-day.",
"corrected_text": "to-day!",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to-day.",
"corrected_text": "to-day!",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Marseilles,",
"corrected_text": "Marseilles.",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "allong",
"corrected_text": "along",
"observed_line_number": 45,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "marshong",
"corrected_text": "marshong.",
"observed_line_number": 45,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to-day.",
"corrected_text": "to-day!",
"observed_line_number": 49,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "to-day.",
"corrected_text": "to-day!",
"observed_line_number": 51,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:23.147514",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:21:11.297865",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "my poor child",
"corrected_text": "my poor child,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the warmth",
"corrected_text": "the warmth;",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "anything else",
"corrected_text": "anything else,",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine",
"corrected_text": "why did he dine to",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the cold stones",
"corrected_text": "the cold stones?",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:26.415602",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:46:13.551162",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "my child",
"corrected_text": "my child.",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "csp",
"corrected_text": "CSP",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine",
"corrected_text": "why did he dine to",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the cold stones",
"corrected_text": "the cold stones?",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:40.794895",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-08T06:32:11.972436",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "my child",
"corrected_text": "my child,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "why did he dine",
"corrected_text": "why did he dine to",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "the cold stones",
"corrected_text": "the cold stones?",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:37.810570",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:22:07.060709",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 38,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"capitalization\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase \"if he\" \u2192 \"If he\". The prediction capitalizes the same word but with less surrounding context: \"if\" \u2192 \"If\". This still captures the core intended change (capitalizing \"if\" to \"If\"), and omitting the following word \"he\" does not alter the essential correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "said",
"corrected_text": "Said",
"observed_line_number": 41,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:42.197219",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:47:04.719963",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 34,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the word \"if\" at the start of the phrase \"if he\" (\"if he\" \u2192 \"If he\"). The prediction captures the same core change by capitalizing \"if\" (\"if\" \u2192 \"If\"). Although it includes less surrounding context (omits \"he\"), it still represents the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 38,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:35.007015",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:23:03.856483",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said (",
"corrected_text": "said, (",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Clennam,",
"corrected_text": "Clennam in",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:44.662218",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:47:58.835026",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said (",
"corrected_text": "said, (",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Clennam,",
"corrected_text": "Clennam in",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:45.280625",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-08T06:33:59.296596",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "high-road",
"corrected_text": "high road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said (",
"corrected_text": "said, (",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Clennam,",
"corrected_text": "Clennam in",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:54:56.269854",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:23:55.116198",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:00.396539",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:48:40.146156",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": ".",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:10.056693",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-08T06:34:52.942600",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:20.442720",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:24:49.017255",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "trees,",
"corrected_text": "trees;",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "with wet; limping",
"corrected_text": "with wet; limping,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "knives!",
"corrected_text": "knives!",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "darkness,",
"corrected_text": "darkness;",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose ,",
"corrected_text": "Let Loose,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone.",
"corrected_text": "Sa\u00f4ne.",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "hyphen",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "soddened",
"corrected_text": "sodden",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:08.209305",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:51:41.342250",
"details": [
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:12.061012",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:25:44.219166",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "heart,",
"corrected_text": "heart;",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "him from her heart",
"corrected_text": "him from her heart,",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hoped so",
"corrected_text": "I hope so",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that! I hope so",
"corrected_text": "that! I hope so;",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:10.444129",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:52:41.761170",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "heart,",
"corrected_text": "heart;",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "him from her heart (as",
"corrected_text": "him from her heart; (as",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hoped so",
"corrected_text": "I hope so",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that\u2019s well! I hope so; I am not disappointed.",
"corrected_text": "that\u2019s well! I hope so; I am not disappointed.",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:17.387860",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-08T06:36:37.088233",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "indeed",
"corrected_text": "indeed,",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hoped so",
"corrected_text": "I hope so",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "heart,",
"corrected_text": "heart, so",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I hope so",
"corrected_text": "I hope so;",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:25.034552",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:26:39.856370",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "rose or two from hers",
"corrected_text": "rose or two from her",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters",
"corrected_text": "on her matters",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment, with",
"corrected_text": "moment; with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that papa had abandoned",
"corrected_text": "that Papa had abandoned",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they are to be married.",
"corrected_text": "They are to be married.",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "confidence, if",
"corrected_text": "confidence; if",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:31.449826",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:53:33.734405",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "rose or two from hers",
"corrected_text": "rose or two from her",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and out at that arch of light at",
"corrected_text": "and out at that arch of light, at",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "for a moment, with",
"corrected_text": "for a moment; with",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that papa had abandoned",
"corrected_text": "that Papa had abandoned",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "confidence",
"corrected_text": "Confidence",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "they are to be married.",
"corrected_text": "they are to be married.",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:32.097277",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:27:33.818120",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in our fancy",
"corrected_text": "that in our fancy,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors.",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "than once",
"corrected_text": "than once,",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:32.473902",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:54:30.907334",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in our fancy",
"corrected_text": "that in our fancy,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "than",
"corrected_text": "then",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:33.453156",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-08T06:38:24.570227",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that in our fancy",
"corrected_text": "that in our fancy,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within-doors.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "than once",
"corrected_text": "than once,",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:47.431314",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:28:33.763719",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the",
"corrected_text": "The",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them:",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence; at",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it;",
"corrected_text": "it:",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "when",
"corrected_text": "when,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view;",
"corrected_text": "view,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence;",
"corrected_text": "existence:",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege lord,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:56:01.083268",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:55:29.012926",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "the",
"corrected_text": "The",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "each",
"corrected_text": "each ",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them:",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence; at",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it;",
"corrected_text": "it:",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "when",
"corrected_text": "when,",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege lord,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "c",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence;",
"corrected_text": "existence:",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view;",
"corrected_text": "view,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:50.149338",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:29:28.292982",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of the driest snorts.",
"corrected_text": "of the driest snorts.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of",
"corrected_text": "benevolence bubbling out of",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:54.436805",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:56:22.349350",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of the driest snorts.",
"corrected_text": "of the driest snorts.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of",
"corrected_text": "benevolence bubbling out of",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:55.407037",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-08T06:40:12.122938",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of the driest snorts.",
"corrected_text": "of the driest snorts.",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorise",
"corrected_text": "authorize",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "your hands.",
"corrected_text": "your hands.",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:55.406245",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T20:30:25.772397",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "and a faded check too",
"corrected_text": "and a faded check too,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me, not that",
"corrected_text": "me; not that",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fled Arthur",
"corrected_text": "fled, Arthur",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Cucumber",
"corrected_text": "Cucumber,",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:55:59.366735",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-07T21:57:17.282387",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "and a faded check too",
"corrected_text": "and a faded check too,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me, not that",
"corrected_text": "me; not that",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fled Arthur",
"corrected_text": "fled, Arthur",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Cucumber",
"corrected_text": "Cucumber,",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_gpt_4o_mini",
"date": "2026-02-22T15:56:02.218007",
"annotator": "GPT-4o-mini (OR)",
"annotation_date": "2025-04-08T06:41:18.552567",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "and a faded check",
"corrected_text": "and a faded check too",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "me, not that",
"corrected_text": "me; not that",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fled Arthur",
"corrected_text": "fled, Arthur",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Cucumber",
"corrected_text": "Cucumber,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2024-07-18",
"release_source": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/",
"release_notes": "OpenRouter route to GPT-4o mini."
},
{
"model_name": "GPT-5",
"model_id": "gpt-5",
"date": "2026-02-18T01:39:58.336186",
"shots": 2,
"config": {
"model_id": "gpt-5",
"display_name": "GPT-5",
"shots": 2,
"temperature": 0.0,
"date": "2026-02-18",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:01:34.908870",
"file_results": [
{
"model_name": "gpt-5",
"date": "2026-02-18T01:20:53.891800",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:01:34.908870",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit aligns perfectly with the ground truth edit. Firstly, the edit type is labeled as 'punctuation' in both the ground truth and the predicted edit, ensuring edit type accuracy. Secondly, the text content accuracy criterion is met since the core punctuation change, which is the addition of a period after 'Fellow Travellers', is precisely captured in the predicted edit. The start state 'Fellow Travellers' and the end state 'Fellow Travellers.' are identical in both the ground truth and prediction, reflecting the intention of the punctuation correction. Therefore, since both criteria for correct prediction are satisfied, the prediction is considered accurate."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. Firstly, the edit type is 'punctuation' in both the ground truth and the prediction, which matches exactly, satisfying the requirement for edit type accuracy. Secondly, the predicted text content accurately represents the core change from 'howling over' to 'howling, over', maintaining the integrity of the change shown in the ground truth. Therefore, the essential edit of adding a comma after 'howling' is correctly captured. Both criteria of edit type and text content are met, validating that the prediction is correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit meets both evaluation criteria as specified. \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edit specify the type as 'punctuation'. This matches exactly, confirming that the edit type is accurate. \n\n2. Text Content Accuracy: The essential change in both the ground truth and predicted edit is the insertion of a comma in 'yonder to-day,' changing it to 'yonder, to-day'. The prediction correctly captures this core change. There is no discrepancy between the ground truth and prediction in terms of the text change. Both versions reflect exactly the same alteration, confirming the text content accuracy.\n\nSince both criteria are perfectly satisfied, the prediction is deemed correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit on two main criteria. First, the edit type is 'punctuation' for both predicted and ground truth, indicating they are both of the same intended modification type. Second, the text content accuracy criterion is met because the core change, which is the insertion of a comma after 'pockets' in the phrase 'pockets and', is correctly reflected in both the predicted and ground truth edit. While the line number differs, the instruction was to ignore line numbers, and therefore the prediction correctly reflects the intended punctuation edit without any discrepancies concerning the actual text change."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir, is",
"corrected_text": "sir; is",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:21:22.472211",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:31:24.265601",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit accurately based on both criteria. \n\n1. Edit Type Accuracy: The edit type is labeled as 'punctuation' in both the ground truth and the predicted edit, indicating a correct classification of the type of change made. \n\n2. Text Content Accuracy: The original text 'yonder to-day' and the corrected text 'yonder, to-day' are both identical in the ground truth and the prediction. This shows that the CORE change, the addition of a comma after 'yonder', is correctly captured in the prediction. Therefore, the prediction accurately reflects the intention of the ground truth edit. \n\nSince both the edit type and the text content align precisely with the ground truth, the prediction is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 16,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit correctly matches the ground truth edit based on both evaluation criteria. \n\n1. Edit Type Accuracy: The edit type in both the prediction and ground truth is 'punctuation'. This matches exactly, satisfying the requirement for edit type accuracy.\n\n2. Text Content Accuracy: The core change identified by both the ground truth and the prediction is the insertion of a comma between 'pockets' and 'and', transforming 'pockets and' into 'pockets, and'. The predicted edit captures this change accurately, which means the essential modification is correctly reflected.\n\nAs both the type and the actual change in text are correct, the prediction is deemed correct."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:21:53.602736",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:00:04.680135",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. For 'Edit Type Accuracy,' both the ground truth and the predicted edit identify the change as a 'punctuation' edit, and they are consistent in this categorization. For 'Text Content Accuracy,' the specific change from 'yonder to-day' to 'yonder, to-day' is accurately reproduced in the predicted edit. This captures the core punctuation change of adding a comma after 'yonder,' as intended in the ground truth. Therefore, the prediction is correct as it matches both the type and the essential text content change stipulated by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is correct based on the given criteria. \n\n1. **Edit Type Accuracy:** \n - The edit type in the predicted edit is 'capitalization', which matches exactly with the ground truth edit type. Therefore, the edit type is accurately captured.\n\n2. **Text Content Accuracy:** \n - Both the ground truth and the predicted edit involve changing 'Sir' to 'sir', capturing the essential change in capitalization. \n - There are no additional words included in either the original or corrected text in the predicted edit that diverge from the core change identified in the ground truth.\n\nSince both the edit type and the core textual change match the ground truth, the predicted edit is considered correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ";",
"corrected_text": ",",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:22:18.017621",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:16:21.441474",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the type as 'capitalization,' which matches the ground truth edit type. For text content accuracy, the ground truth corrects the phrase 'why did he dine' to 'Why did he dine,' with the core change being the capitalization of the first word 'why' to 'Why.' The predicted edit focuses on this core change by specifying the change from 'why' to 'Why,' leaving out the surrounding context. Both criteria are satisfied as the essential capitalization change is accurately captured in the prediction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"",
"corrected_text": "'",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"",
"corrected_text": "'",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:22:49.881394",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T01:18:55.961869",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "why",
"corrected_text": "\"why",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones!\"",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:23:15.132111",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:03:27.271028",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captured the intended changes as specified in the ground truth. For Edit Type Accuracy, both the ground truth and the predicted edit identified the change as 'capitalization,' which matches exactly. For Text Content Accuracy, the core change in the ground truth was capitalizing the first word 'why' to 'Why.' The predicted edit also identified the essential capitalization change from 'why' to 'Why.' Although the predicted edit isolated just the single word rather than the whole phrase 'why did he dine,' it still effectively captured the required capitalization change. Since both the edit type and the essential text content change were identified correctly, the prediction is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit. For Edit Type Accuracy, both the ground truth and the predicted edit have the same type of edit, which is 'punctuation'. This matches the requirement exactly. For Text Content Accuracy, the essential change is transforming the punctuation from an exclamation mark '!' to a question mark '?'. The ground truth specifies the change in the phrase 'cold stones!', whereas the prediction specifies 'stones!'. Despite the additional word 'cold' in the ground truth, the core improvement (the change in punctuation) is correctly captured in the predicted edit. Therefore, the prediction accurately reflects the required correction and is considered correct."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:23:45.483801",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:18:48.021286",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth in terms of both edit type and text content. \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edit identify the change as a 'capitalization' change. Therefore, this criterion is satisfied. \n\n2. Text Content Accuracy: The core change identified in the ground truth is the capitalization of 'if' to 'If'. The predicted edit captures this same core change. While the ground truth includes 'he' for additional context, the essential edit which is the capitalization of 'if' is accurately identified by the prediction. The additional words in the ground truth (specifically 'he') do not affect the correctness of the core edit, which is correctly recognized as changing 'if' to 'If'. \n\nThus, the predicted edit correctly captures the intention of the ground truth action."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth in terms of edit type and essential text content. \n\n1. Edit Type Accuracy: Both the ground truth and the prediction identify the edit type as 'replacement.' This is a correct match as there is no discrepancy in the classification of the type of edit between the two.\n\n2. Text Content Accuracy: The core change in both the ground truth and prediction is 'said' being replaced by 'asked.' The prediction captures this essential change even though it does not include the additional contextual word 'Clennam' in its identified original and corrected texts. The essential transformation of 'said' to 'asked' is accurately captured, which aligns with the objective of the editorial correction. \n\nTherefore, the prediction is considered correct as it fulfills both evaluation criteria."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:24:14.005566",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:37:09.440539",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit and the ground truth edit both specify 'capitalization' as the edit type, which matches exactly and satisfies the Edit Type Accuracy criterion. For Text Content Accuracy, both edits capture the core change of capitalizing the word 'if' to 'If', which is the essential change indicated in the ground truth. Although the ground truth includes additional context by listing 'if he' as the original text while the prediction lists just 'if', the essential change\u2014capitalization of the word 'if'\u2014is accurately and effectively captured by the prediction. Therefore, both criteria are met, and the predicted edit is deemed correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both the ground truth and predicted edit share the same type of edit, which is a 'replacement'. Therefore, the edit type accuracy criterion is satisfied. \n\n2. Text Content Accuracy: The core change identified in the ground truth is the conversion from 'said' to 'asked'. The predicted edit successfully captures this essential change. While the context in the ground truth includes 'Clennam', the core replacement targeted by the prediction aligns with the desired modification by converting 'said' to 'asked'. Therefore, the text content accuracy criterion is also satisfied. \n\nBoth criteria are met, so the prediction is deemed correct."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:24:41.990881",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:05:50.102358",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the 'capitalization' type, which matches the ground truth edit type. Regarding the text content accuracy, the ground truth specifies the change from 'if he' to 'If he', focusing on capitalizing the first word 'if' to 'If'. The prediction specifies the change from 'if' to 'If', which also correctly highlights the core change of capitalizing 'if'. Although the ground truth includes the additional word 'he' in the context, the essential edit is the capitalization of 'if', and the prediction successfully captures this. Therefore, both the edit type and the text content accuracy check are satisfied."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:25:15.008333",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:21:53.402485",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is 'punctuation', which matches exactly with the ground truth edit type 'punctuation'. This criterion is satisfied. 2. Text Content Accuracy: The ground truth indicates a change from 'high road' to 'high-road'. The predicted edit captures the same core change, accurately reflecting 'high road' to 'high-road'. There is no additional context that affects the core change. Therefore, the text content accuracy criterion is also satisfied. Overall, both criteria are met, so the prediction is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria laid out for evaluation. Firstly, the edit type is 'punctuation' in both the ground truth and the prediction, so the edit type accuracy criterion is fully satisfied. Secondly, the text content accuracy criterion requires that the core change identified in the ground truth is captured in the prediction. The ground truth edit shows a correction from 'about and' to 'about, and', and the predicted edit accurately reflects this change by including the comma after 'about'. Thus, the primary modification, which is the insertion of a comma, is correctly captured. Therefore, both the edit type and the core textual change align perfectly between the ground truth and the predicted edit, confirming the prediction as correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The evaluation consists of two main criteria: Edit Type Accuracy and Text Content Accuracy. \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edit have the type 'punctuation'. There is an exact match, which satisfies the criteria for edit type accuracy. \n\n2. Text Content Accuracy: The core correction intended by the ground truth is to add a comma after 'Clennam' in the phrase 'Clennam in', converting it to 'Clennam, in'. The predicted edit correctly captures this exact change from 'Clennam in' to 'Clennam, in'. There are no additional changes or errors. \n\nGiven that both criteria are met, the predicted edit is accurate according to the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:25:54.953855",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:39:19.372963",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria specified. \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edit are categorized as 'punctuation'. This indicates that the nature of the correction, focusing on adjusting punctuation within the text, is accurately represented in both.\n\n2. Text Content Accuracy: The change specified in both the ground truth and the prediction involves altering 'high road' to 'high-road'. This correction focuses on the addition of a hyphen between 'high' and 'road'. Both the ground truth and predicted edit accurately capture this transformation, confirming that the core textual change is correctly identified. Thus, the predicted edit accurately reflects the intention of the ground truth edit, and is therefore correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about,and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth in both criteria: edit type and text content. \n\n1. Edit Type Accuracy: The predicted edit specifies 'punctuation' as the type, which aligns exactly with the ground truth's 'punctuation' edit type. This indicates that, in terms of edit classification, the prediction is correct.\n\n2. Text Content Accuracy: The core change represented in the ground truth is the insertion of a comma in 'about and' to 'about, and'. The predicted edit captures the same core change by transforming 'about,and' to 'about, and'. The minor difference in the original text format ('about and' vs. 'about,and') does not affect the essential change, which is the insertion of the comma. Therefore, the essential edit is captured accurately by the prediction.\n\nGiven these observations, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association",
"corrected_text": "association,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:26:32.533380",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:08:56.026825",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both key aspects: edit type and text content. Firstly, the edit type 'punctuation' is the same for both the ground truth and the predicted edit, which satisfies the requirement for edit type accuracy. Secondly, the text content change from 'high road' to 'high-road' is accurately captured in the predicted edit, matching the ground truth's core change. Given that both the edit type and the essential change in text are correctly understood and represented, the prediction is correct according to the evaluation criteria."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said,when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the type as 'punctuation', which matches the ground truth edit type. For text content accuracy, the ground truth changes 'said when' to 'said, when', indicating the insertion of a comma between 'said' and 'when'. The predicted edit starts with 'said,when' and changes it to 'said, when'. Despite the initial lack of a space after the comma in the original text of the predicted edit, the core change\u2014introducing a space after the comma\u2014is correctly captured, aligning the original and corrected texts with the ground truth intention. Therefore, both criteria for edit type and text content are met, resulting in a correct evaluation."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit on both key criteria. Firstly, the edit type is 'punctuation' for both the ground truth and the prediction, which means the intended change involves adjusting punctuation and they match on this aspect. Secondly, the text content accuracy is maintained as the core change identified in both the ground truth and the prediction is the addition of a comma after the word 'said', changing it from 'said' to 'said,'. Both the ground truth and predicted text indicate the same transformation, and thus the essential edit is correctly captured. Since both criteria are satisfied, the prediction is accurate."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly matches the ground truth edit in both criteria. \n\n1. **Edit Type Accuracy**: The edit type in the predicted edit is 'punctuation', which aligns perfectly with the ground truth edit type of 'punctuation'. Thus, the edit type requirement is satisfied.\n\n2. **Text Content Accuracy**: The essential change in the ground truth is the insertion of a comma after 'Clennam', changing 'Clennam in' to 'Clennam, in'. The predicted edit captures this core change by altering 'Clennam' to 'Clennam,'. While the predicted edit does not mention 'in', it correctly identifies the essential punctuation change. Therefore, the text content change is accurately captured.\n\nIn conclusion, as both the edit type and the core textual change are correctly addressed by the prediction, the edit is considered accurate."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tion,did",
"corrected_text": "tion, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about,and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:26:58.122010",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:23:11.695779",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:27:23.864632",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T01:19:43.332982",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:27:48.245512",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:10:06.761737",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:28:29.489201",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:26:33.968953",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit. Firstly, the edit type is 'replacement' in both the ground truth and the predicted edit, satisfying the Edit Type Accuracy requirement. Secondly, with regard to Text Content Accuracy, both the original text 'Saone' and the corrected text 'Sa\u00f4ne' in the prediction match the ground truth exactly. The core change involving replacing 'Saone' with 'Sa\u00f4ne' is correctly captured by the prediction. Therefore, the predicted edit is correct as it meets both criteria, ensuring the change is identified and executed as intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit. First, the edit type matches exactly; both are classified as 'replacement', indicating a substitution of text. This satisfies the Edit Type Accuracy criterion. Second, the Text Content Accuracy is achieved as well: the original text 'Saone' is correctly identified in both the ground truth and predicted edits, and the corrected text 'Sa\u00f4ne' is accurately reflected in the predicted edit. The core change, specifically the substitution of 'Saone' with 'Sa\u00f4ne' (adding the circumflex accent on the 'o'), is precise and matches the ground truth. Therefore, both criteria are met, confirming the prediction is correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Both the predicted edit and the ground truth edit have correctly identified the edit type as 'punctuation', which matches exactly. Regarding text content accuracy, both the ground truth and the prediction specify the change from 'man slowly' to 'man, slowly', which captures the core change of adding a comma after 'man'. Since both the edit type and the essential text content change are correctly captured by the predicted edit, it aligns perfectly with the ground truth's intention. Therefore, the predicted edit is correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "long heavy",
"corrected_text": "long, heavy",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ",",
"corrected_text": "",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "ed",
"corrected_text": "",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:28:56.622080",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:43:24.152913",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit based on the evaluation criteria. \n\n1. Edit Type Accuracy: The predicted edit type is 'replacement', which matches the ground truth edit type. Since the types are identical, the predicted edit meets the requirement for edit type accuracy. \n\n2. Text Content Accuracy: The prediction identifies the core change from 'sodden' to 'soddened', which is precisely what the ground truth specifies. The essential alteration of adding 'ed' to the word 'sodden' is accurately captured in the predicted edit. \n\nThus, both the edit type and text content accuracy are in alignment with the ground truth, leading to a correct prediction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone it",
"corrected_text": "Saone, it",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:29:38.683984",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:13:36.898053",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit in both required dimensions of evaluation. \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edit specify the edit type as 'replacement,' which exactly matches, fulfilling the requirement for edit type accuracy.\n\n2. Text Content Accuracy: In both the ground truth and predicted edits, the original text 'Saone' is replaced with 'Sa\u00f4ne.' The core change involving the inclusion of the circumflex accent in the word is correctly captured in the prediction. Since the focus is on the essential edit, and both 'original_text' and 'corrected_text' fields are identical between the ground truth and the prediction, the text content accuracy is also maintained.\n\nConsidering both criteria being met satisfactorily, the predicted edit aligns perfectly with the ground truth edit's intentions."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The evaluation of the predicted edit against the ground truth edit involves two main criteria: Edit Type Accuracy and Text Content Accuracy. \n\n1. **Edit Type Accuracy:** The type specified in both the ground truth and predicted edit is 'replacement'. There is an exact match between the stated edit types, satisfying this criterion.\n\n2. **Text Content Accuracy:** The ground truth specifies a correction of the word 'Saone' to 'Sa\u00f4ne', indicating the addition of a circumflex accent over the 'o'. The predicted edit also specifies changing 'Saone' to 'Sa\u00f4ne', accurately capturing this core change. The essential edit, which is adding the circumflex accent, is correctly identified and reproduced in the prediction.\n\nAs both criteria are met, the predicted edit perfectly aligns with the ground truth, thus making it correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ",",
"corrected_text": ".",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:30:08.148898",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:30:16.193246",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "And,",
"corrected_text": "And",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart",
"corrected_text": "heart,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:30:48.140743",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:49:29.796337",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit. Firstly, the edit type is 'punctuation' in both cases, which meets the requirement for Edit Type Accuracy. Secondly, regarding Text Content Accuracy, the core change from the original text 'heart so' to the corrected text 'heart, so' is identical in both the ground truth and predicted edit. The insertion of a comma between 'heart' and 'so' is correctly identified and applied, meaning that the essential punctuation change is accurately captured. Since both criteria are satisfied, the prediction is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit accurately in both criteria. \n\n1. Edit Type Accuracy: Both the ground truth and predicted edit specify 'punctuation' as the type of edit. This suggests that the nature of the change is correctly identified as a punctuation-related correction. \n\n2. Text Content Accuracy: The ground truth and the prediction both focus on the same text change from 'length said' to 'length, said'. The core change, which is the addition of a comma after 'length', is accurately captured in the prediction. Even if additional text context were included, the essential change would still be correctly represented. \n\nBoth criteria are met, confirming the predicted edit captures the ground truth's intention correctly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately matches the ground truth in both criteria. \n\n1. **Edit Type Accuracy**: The predicted edit type is 'replacement', which matches the ground truth edit type. Therefore, it fulfills the requirement for matching the edit type exactly.\n\n2. **Text Content Accuracy**: The core change involved in both the ground truth and the prediction is the transformation from 'hope' to 'hoped'. Although the ground truth includes additional context ('I hope so' to 'I hoped so'), the essential change\u2014the verb tense alteration from present to past\u2014is correctly captured by the prediction. The focus is on identifying the essential transformation of 'hope' to 'hoped', which the prediction does correctly, making it consistent with the ground truth's intention.\n\nSince both the type and the core content of the change align with the ground truth, the predicted edit is considered correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "some",
"corrected_text": "",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:31:17.367768",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:16:41.024127",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "became in",
"corrected_text": "became, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "heart",
"corrected_text": "heart indeed",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "(as she did, if it be the source of earnestness),",
"corrected_text": "",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:31:47.828552",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:33:23.705818",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the type and content of the correction in the ground truth. Both the ground truth and predicted edits are of the 'punctuation' type, accurately reflecting the necessary alteration. Additionally, the text content accuracy is upheld as both show the transformation from 'hers matters' to 'hers, matters', capturing the inclusion of the comma. Ignoring line numbers and focusing solely on the type and text, the prediction matches the essential correction the ground truth specifies, thereby meeting all evaluation criteria."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. Both the predicted and ground truth edits have the same edit type, 'punctuation'. Additionally, the text content change is accurately captured in the prediction. The core change involves adding a comma after the word 'moment', changing the text from 'moment with' to 'moment, with'. Since both the edit type and the essential text change are correctly identified, the prediction is deemed correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit accurately on both criteria. Firstly, the edit type is exactly the same as the ground truth, labeled as 'punctuation', which indicates that the punctuation modification is correctly identified. Secondly, the text content change is identical in both the predicted and ground truth edits: the insertion of a comma between 'beautiful' and 'that'. The core change from 'beautiful that' to 'beautiful, that' is correctly identified by the prediction, fulfilling the essential edit requirement. Therefore, the prediction is correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately matches the ground truth in both the edit type and the text content. The edit type is 'capitalization' in both cases, which aligns perfectly. Furthermore, the text content accuracy is also maintained as the core change from 'they' to 'They' is correctly captured in both the ground truth and the prediction. There are no discrepancies in the essential edit, and thus the prediction fully aligns with the ground truth\u2019s intention. Additional context is not present, but it is not necessary given the nature of the correction. Therefore, the prediction is deemed correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "which \u2014 that",
"corrected_text": "which\u2014that",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:32:26.812512",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:54:17.138533",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both type and content. \n\n1. Edit Type Accuracy: The type of edit in both the ground truth and the prediction is 'punctuation'. This matches exactly, so the edit type is correct.\n\n2. Text Content Accuracy: The textual change identified in both the ground truth and the prediction is from 'hers and' to 'hers, and'. The core change, which is the addition of a comma after 'hers', is correctly captured in the prediction. \n\nAs both the edit type and the text content accurately match the ground truth's intention, the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both key criteria. The edit type is labeled as 'punctuation' in both cases, which is the correct classification for the insertion of a comma. Additionally, the text content accuracy is maintained as both the original and the corrected text in the prediction are exactly the same as those in the ground truth. The core change from 'hers matters' to 'hers, matters' is correctly captured in the predicted edit, fulfilling both requirements for a correct evaluation."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is correct because it meets both criteria required for accuracy. Firstly, the edit type matches exactly; both the ground truth and the prediction identify the edit as a 'punctuation' type. Secondly, the text content accuracy is satisfied as well. Both the ground truth and the predicted edit indicate the essential change from 'moment with' to 'moment, with'. The core change, which is the insertion of a comma after 'moment', is correctly captured. Although both versions display the same surrounding text, the primary focus is on the accurate representation of the change, which has been successfully achieved in the prediction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria evaluated. \n\n1. Edit Type Accuracy: \n - The edit type in both the ground truth and the predicted edit is 'punctuation'. Therefore, they match exactly. \n\n2. Text Content Accuracy: \n - The core change in text content is the addition of a comma: 'beautiful that' is corrected to 'beautiful, that'. Both the ground truth and predicted edit capture this punctuation change accurately. Since the essential change is correctly identified in both the original and corrected texts, the prediction aligns with the ground truth.\n\nSince both the edit type and text content match the ground truth's intention, the prediction is considered correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth in both edit type and text content. For Edit Type Accuracy, both the ground truth and the prediction have the edit type as 'capitalization', which matches perfectly. For Text Content Accuracy, the core change is the capitalization of the word 'they' to 'They'. The predicted edit captures this exact change, mirroring the core alteration identified in the ground truth. Additionally, both versions refer to the same page and disregard the specific line number as instructed. Therefore, the prediction is entirely correct as per the evaluation criteria."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:33:00.947304",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:18:29.364079",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both the type and the text content. The edit type 'punctuation' is exactly the same in both the ground truth and the predicted edit. The text content focuses on the addition of a comma after 'hers' to change 'hers and' to 'hers, and'. Both the ground truth and the prediction reflect this same core change accurately, regardless of any additional context. Hence, the predicted edit fulfills both criteria for correctness: the edit type matches, and the essential text change is captured precisely as intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is evaluated based on two criteria: Edit Type Accuracy and Text Content Accuracy. For Edit Type Accuracy, both the ground truth and the predicted edit classify the change as a 'punctuation' edit. This matches exactly, so the edit type is correct. For Text Content Accuracy, the ground truth indicates the original text 'hers matters' should be corrected to 'hers, matters'. The predicted edit exactly mirrors this text change with 'hers matters' becoming 'hers, matters'. Therefore, the core change in punctuation is accurately captured. Because both the edit type and the text content criteria are satisfied, the predicted edit is correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both the essential criteria: edit type and text content accuracy. \n\n1. Edit Type Accuracy: Both the predicted and ground truth edits are of type 'punctuation.' This matches exactly, fulfilling the requirement for edit type accuracy. \n\n2. Text Content Accuracy: The core change is the insertion of a comma after the word 'moment' to correct the text from 'moment with' to 'moment, with.' The prediction captures this change precisely, addressing the punctuation correction intended by the ground truth. \n\nSince both criteria are satisfied, the predicted edit is considered correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth in both critical aspects required for an accurate classification. \n\n1. Edit Type Accuracy: The predicted edit type is 'punctuation', which exactly matches the ground truth's type. This alignment means the prediction correctly understood the nature of the edit being related to punctuation. \n\n2. Text Content Accuracy: The core change, to insert a comma in the phrase 'beautiful that' making it 'beautiful, that', is precisely captured in both the predicted and ground truth edits. Even if the prediction included more or less context, the essential change of adding a comma is accurately identified in both the original and corrected text portions. \n\nGiven these considerations, the prediction correctly interprets the core editorial change intended by the ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly matches the ground truth edit on both the edit type and the text content. The edit type is 'capitalization' in both cases, indicating that the change pertains to capitalizing the first letter of the word. The original and corrected texts match exactly in both the ground truth and the prediction, with the correction of 'they' to 'They'. Therefore, the prediction accurately captures the essential change identified by the ground truth."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:33:39.338571",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:36:44.144173",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit based on the criteria provided. \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edit have the edit type as 'replacement'. This is an exact match, satisfying the requirement that the edit types must align exactly. \n\n2. Text Content Accuracy: The essential change in both edits is the replacement of 'that' with 'than'. The predicted edit captures this core change accurately, which aligns with the ground truth's intent. Additional context or line numbers are ignored per instructions, and thus do not affect the correctness of the predicted edit. \n\nSince both the edit type and text content accurately reflect the ground truth's intention, the prediction is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore,",
"corrected_text": "shore",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:34:15.858344",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T01:00:35.495898",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 18,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit correctly matches the ground truth edit in terms of edit type and text content. Both specify the change type as 'punctuation', indicating the intent to modify punctuation. The core textual change in both predictions is the addition of a comma after 'Thank you'. While the original text in the ground truth includes an exclamation mark ('Thank you!') and the prediction does not ('Thank you'), the key focus is whether the prediction captures the introduction of the comma. In terms of content, the essential edit required, which is adding a comma after 'Thank you', is accurately identified by the prediction. Thus, the prediction aligns with the intention of the ground truth, despite the slight variation in the original text context."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit and the ground truth edit both specify the type as 'punctuation', which matches exactly. Additionally, the text content accuracy criterion is met because the essential change from 'shore and' to 'shore, and' is correctly identified by both the predicted edit and the ground truth. The core change involves the insertion of a comma, which is accurately captured in the predicted edit, notwithstanding the difference in line number, which is to be ignored in this evaluation. Therefore, both criteria for evaluation are satisfied, confirming the accuracy of the prediction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 34,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit matches the ground truth edit in both key areas of evaluation. \n\n1. Edit Type Accuracy: The type of edit is 'replacement' for both the ground truth and the prediction. This indicates that the prediction correctly identifies that a replacement needs to occur, thus meeting this criterion. \n\n2. Text Content Accuracy: The core change identified by the ground truth is a replacement of 'that' with 'than'. The predicted edit accurately captures this same essential change, replacing 'that' with 'than' as well. Despite any additional context, such as line numbers or additional text which are to be ignored in this evaluation, the exact replacement matches the ground truth's intention. \n\nTherefore, based on the above analysis, both criteria for a correct prediction are satisfied, and the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:34:50.224769",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:20:27.249772",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit and the ground truth edit both have the same type, 'punctuation', indicating that they are identified as the same category of edit. For text content accuracy, the ground truth edit involves changing 'Thank you!' to 'Thank you,'. The predicted edit identifies 'Thank you' as the original text and correctly changes it to 'Thank you,'. The essential change here is the replacement of or addition after 'Thank you' where a punctuation is shifted from '!' to ','. Although the original text in the prediction omitted the '!', focusing on the corrected text 'Thank you,' shows that the core intention of adding a comma has been captured accurately. Thus, both criteria are satisfied, making the prediction correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both essential criteria. The edit type is 'punctuation' in both the ground truth and prediction, ensuring edit type accuracy. Additionally, the core change in text content is 'shore and' to 'shore, and' in both cases. The essential edit, which is the insertion of a comma, is correctly captured in the prediction. Therefore, the prediction accurately captures the intention of the ground truth edit in terms of both edit type and text content."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit perfectly in terms of both criteria. Firstly, the 'Edit Type Accuracy' is satisfied because both the ground truth and the predicted edit specify a 'replacement' type. Secondly, the 'Text Content Accuracy' is also met because the core change from 'that' to 'than' is correctly identified in both the ground truth and the prediction. The original and corrected texts in the prediction exactly match those in the ground truth, capturing the essential edit without any discrepancies. Since both the edit type and the text content are accurate in the prediction, the predicted edit is considered fully correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:35:26.480389",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:40:02.067491",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both essential criteria. First, the Edit Type Accuracy is satisfied because both the ground truth and predicted edits are classified as 'capitalization'. This edit type requires changing the case of a letter, and both texts reflect this requirement. Second, the Text Content Accuracy is also matched because the change in text content is identical. The original text 'city' is changed to 'City' in both cases, capturing the core change intended by the ground truth. Both edits focus on the capitalization of the first letter of the word 'city', which is essential to the correction process. Since both criteria are met accurately, the predicted edit is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is considered correct. Firstly, the edit type 'punctuation' matches exactly between the ground truth and the prediction, satisfying the criterion for Edit Type Accuracy. Secondly, regarding Text Content Accuracy, the core edit in the ground truth is the addition of a comma after 'existence,' changing 'existence' to 'existence,'. The predicted edit also correctly captures this essential change by transforming 'existence' to 'existence,' even though it includes additional context by referring to 'existence at' instead of just 'existence.' According to the guidelines, the inclusion of additional context does not invalidate the correct capture of the essential edit. Therefore, the prediction accurately reflects the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it;",
"corrected_text": "it,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence;",
"corrected_text": "existence,",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liegelord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each",
"corrected_text": "to each",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,",
"corrected_text": "view",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:36:09.705022",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T01:04:23.151599",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit has successfully captured both the edit type and the text content change as outlined in the ground truth. \n\n1. **Edit Type Accuracy:** \n - The edit type in both the predicted and ground truth edits is 'capitalization', and they match exactly. Therefore, the edit type criterion is satisfied.\n\n2. **Text Content Accuracy:** \n - The original text 'city' is correctly transformed to the corrected text 'City', which accurately reflects the core change from lowercase to uppercase. This indicates that the essential change intended by the ground truth has been captured in the prediction.\n\nOverall, since the predicted edit matches both aspects required for accuracy as described, the evaluation results in a correct judgment."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth on both key criteria. Firstly, the edit type is correctly identified as a 'replacement' for both the ground truth and the predicted edit. Secondly, the text content accuracy criterion is also met: the change from 'of' to 'to' is exactly the same in both the ground truth and predicted edits. There is no discrepancy in the core change intended by the ground truth. Therefore, the prediction is considered correct as it fully aligns with the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit by meeting both required criteria. Firstly, the edit type 'punctuation' is an exact match between the ground truth and the prediction, satisfying the Edit Type Accuracy requirement. Secondly, for Text Content Accuracy, the essential change involves adding a comma to the word 'existence'. The prediction makes the same core alteration by suggesting 'existence' be changed to 'existence,'. Thus, it accurately reflects the fundamental punctuation change anticipated in the ground truth without any discrepancies. As both criteria are satisfied, the predicted edit is deemed correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit based on the provided criteria. Firstly, the edit type is the same for both the ground truth and the prediction: 'punctuation'. This matches exactly, meeting the requirement for edit type accuracy. Secondly, the text content accuracy is also fulfilled because the core change from 'view' to 'view,' is the same in both the ground truth and the prediction. The essential edit, which is the addition of a comma after the word 'view', has been correctly captured. This evaluation is not concerned with line numbers, and since both the type and text change align perfectly with the ground truth, the prediction is correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:36:43.833860",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:22:06.715431",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria. Firstly, the edit type 'capitalization' in the predicted edit is identical to the ground truth, satisfying the requirement for matching edit types. Secondly, the text content is accurately captured: both the original text 'city' and the corrected text 'City' are exactly the same in the prediction as in the ground truth. The essential change, which is capitalizing the word 'city' to 'City', is accurately captured by the prediction, aligning perfectly with the intent of the ground truth edit. Since both criteria are met, the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of",
"corrected_text": "return to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth in terms of edit type, as both are 'replacement'. Text content accuracy is also met. The core change intended by the ground truth is replacing 'of' with 'to'. The prediction captures this essential change within the phrase 'return of' to 'return to'. The additional context included in the prediction does not affect the accuracy, as the central change intended by the ground truth (replacing 'of' with 'to') is clearly preserved."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit meets both the required criteria. \n\n1. Edit Type Accuracy: The predicted edit and the ground truth edit both specify the type as 'punctuation'. This is an exact match, so the criterion is satisfied.\n\n2. Text Content Accuracy: The core change in the text is from 'existence' to 'existence,'. The predicted edit captures this specific change. While it includes additional context ('existence at' \u2192 'existence, at'), the essential alteration, which is the addition of a comma after 'existence,' is accurately reflected. The additional context does not alter the core change intention specified by the ground truth, so this criterion is also satisfied.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is evaluated based on two criteria: edit type accuracy and text content accuracy. \n\n1. **Edit Type Accuracy**: Both the ground truth and predicted edits have the type 'punctuation'. This matches exactly, fulfilling the requirement for edit type.\n\n2. **Text Content Accuracy**: The ground truth edit involves changing 'view' to 'view,'. The predicted edit changes 'view by' to 'view, by', which captures the same core punctuation change: the insertion of a comma after 'view'. While the prediction includes additional words 'by', it accurately reflects the same dedication towards adding a comma after 'view'. Thus, the essential modification has been correctly identified and implemented.\n\nOverall, the predicted edit satisfies both the criteria set forth: it has the correct edit type and captures the core textual change accurately. Therefore, the predicted edit is considered correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:37:20.109014",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:44:15.988074",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the edit type as 'italicize,' which matches the ground truth edit type. Regarding text content accuracy, the core change in the ground truth is italicizing 'him.' The predicted edit captures this by including the same word 'him' for italicization, even though it includes additional context with the surrounding words 'benevolence bubbling out of.' Since it successfully identifies the essential text to be italicized ('him'), while containing additional non-essential text, the prediction matches the ground truth intentions. Therefore, both the edit type and core change are accurately captured in the prediction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is correct based on the criteria provided. \n\n1. Edit Type Accuracy: The edit type in both the ground truth and the predicted edit is 'replacement'. This matches exactly, fulfilling the requirement for this criterion. \n\n2. Text Content Accuracy: The predicted edit shows the replacement of 'authorize' with 'authorise', which perfectly captures the core change identified in the ground truth. Both the original and corrected texts match exactly between the ground truth and predicted edit. Since both criteria are met, the prediction is considered correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "series",
"corrected_text": "series of",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "the",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:37:56.852093",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T01:08:41.565486",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the type of edit as 'italicize,' matching the ground truth. Both ground truth and predicted edits involve italicizing text, which ensures edit type accuracy. For text content accuracy, the ground truth focuses on italicizing the word 'him,' while the prediction includes the broader context 'benevolence bubbling out of him,' encompassing the specific word being italicized. Although the prediction provides additional context, it accurately captures the core change noted in the ground truth by italicizing 'him.' Thus, both criteria for a correct evaluation are met, making the prediction accurate."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the ground truth edit based on the specified criteria. For Edit Type Accuracy, both the predicted edit and the ground truth edit are of type 'replacement', indicating a direct substitution of text, which matches exactly. Regarding Text Content Accuracy, the core change identified by the ground truth edit is the replacement of 'authorize' with 'authorise'. The predicted edit makes the exact same change, capturing the essential part of the edit accurately. Since both criteria\u2014Edit Type Accuracy and Text Content Accuracy\u2014are met, the predicted edit is correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "the",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:38:28.782642",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:24:04.459050",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 28,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "The predicted edit correctly identifies the edit type as 'italicize', which matches the ground truth edit type. Regarding text content accuracy, the predicted edit includes the full phrase 'benevolence bubbling out of him' instead of just 'him'. However, it retains the italicization change intended for 'him', which is the core change identified in the ground truth. This inclusion of additional context does not alter the intended italicization of 'him'. Therefore, since both the edit type and the essential text change align with the ground truth, the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": 32,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. Both the predicted and ground truth edits have the same type: 'punctuation', so the Edit Type Accuracy requirement is satisfied. In terms of Text Content Accuracy, the core change from 'now I' to 'now, I' is accurately captured in the prediction. The predicted text content reflects the same essential change as indicated by the ground truth. As line numbers are disregarded for this evaluation, they do not affect the correctness of the prediction. Therefore, both the edit type and text content of the predicted edit match the ground truth, validating the prediction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 35,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit despite the difference in line numbers, which we are ignoring for this evaluation. Both the ground truth and the prediction have the same 'replacement' edit type, which matches the requirement that the edit types must be exactly the same. Furthermore, the 'original_text' of 'authorize' and the 'corrected_text' of 'authorise' are identical in both the ground truth and the predicted edit, capturing the core change intended. Since both the edit type and text content match, the prediction is considered correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "boiling over",
"corrected_text": "boiling-over",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:38:56.178590",
"annotator": "GPT-5",
"annotation_date": "2026-02-17T23:47:22.872811",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "x",
"corrected_text": "",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:39:27.162851",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T01:12:34.984594",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "never",
"corrected_text": "never,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "x",
"corrected_text": "",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me",
"corrected_text": "me,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5",
"date": "2026-02-18T01:39:58.336186",
"annotator": "GPT-5",
"annotation_date": "2026-02-18T00:25:48.227292",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "morning;",
"corrected_text": "morning,",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and,",
"corrected_text": "and",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For,",
"corrected_text": "For",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly,",
"corrected_text": "quickly",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me,",
"corrected_text": "me;",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2025-08-07",
"release_source": "https://openai.com/index/introducing-gpt-5/"
},
{
"model_name": "GPT-5.2 (OR)",
"model_id": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:56.442481",
"shots": 2,
"config": {
"model_id": "or_openai_gpt_5_2",
"display_name": "GPT-5.2 (OR)",
"shots": 2,
"temperature": 0.0,
"date": "2026-02-22",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:29:13.465175",
"file_results": [
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:13.577874",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:29:13.465175",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit on both criteria. First, the edit type is 'punctuation' in both the ground truth and the prediction, satisfying the requirement for Edit Type Accuracy. Second, the Text Content Accuracy criterion is met because the core change of adding a period after 'Fellow Travellers' is accurately captured in both the original and corrected text fields of the prediction. The essential and specific modification ('Fellow Travellers' \u2192 'Fellow Travellers.') is correctly identified. Therefore, the prediction's representation of the change aligns entirely with the intention reflected in the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The evaluation criterion requires that both the edit type and the text content match the ground truth. In this case, both criteria are satisfied: 1. **Edit Type Accuracy**: The predicted edit type is 'punctuation', which matches exactly with the ground truth edit type 'punctuation'. There is no discrepancy here, making the edit type accurate. 2. **Text Content Accuracy**: The original and corrected text in the predicted edit are 'howling over' and 'howling, over', respectively. This aligns precisely with the ground truth's 'original_text' and 'corrected_text'. The core change identified - the addition of a comma after 'howling' - is accurately captured in the corrected text. Since both the type and the content of the edit in the prediction match the ground truth perfectly, the overall prediction is deemed correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit by meeting both key criteria. Firstly, the Edit Type Accuracy is satisfied because both the ground truth and the predicted edit identify the edit as a 'capitalization' type. Secondly, the Text Content Accuracy criterion is met as the core change of transforming 'Sir' to 'sir' is correctly identified in both the original and predicted edits. The essential transformation of changing the capitalization of a word is maintained consistently across both the ground truth and predicted outputs, thereby ensuring the correct outcome. Since both criteria are satisfied, the prediction is considered correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both key evaluation factors. Firstly, the edit type is 'punctuation' in both the ground truth and the prediction, satisfying the requirement for the edit types to match exactly. Secondly, the text content accuracy is confirmed as both the ground truth and the prediction correctly identify the core change, which is the insertion of a comma in 'pockets and' to make it 'pockets, and'. The essential change of adding a comma is captured accurately. Since both the edit type and the text content match, the prediction is considered correct."
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:09.885470",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:29:18.966056",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly matches the ground truth edit in both criteria. Firstly, the edit type in both the ground truth and the prediction is 'punctuation', which is an exact match, satisfying the Edit Type Accuracy requirement. Secondly, the Text Content Accuracy is also fulfilled as both the ground truth and predicted edits indicate the addition of a period after 'Fellow Travellers', changing it from 'Fellow Travellers' to 'Fellow Travellers.'. The core change intended by the ground truth is accurately captured in the prediction. Thus, the prediction successfully replicates the intention of the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria. First, the edit type 'capitalization' is exactly the same in both the ground truth and predicted edit, satisfying the requirement for the edit type accuracy. Second, the core change specified in the text content also aligns perfectly, with both the ground truth and the prediction indicating a change from 'Sir' to 'sir'. Since the essential change is accurately captured (changing the capitalization of the term 'Sir' to 'sir'), the text content accuracy criterion is also met. Therefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit both in terms of type and content. \n\n1. Edit Type Accuracy: The edit type in both the ground truth and the predicted edit is 'punctuation'. This is an exact match, fulfilling the requirement for edit type accuracy.\n\n2. Text Content Accuracy: The essential change described in both the ground truth and the predicted edit is the insertion of a comma, transforming 'pockets and' into 'pockets, and'. The core change is accurately captured in the prediction, satisfying the requirement for text content accuracy.\n\nSince both criteria are correctly matched, the prediction is considered correct."
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:10.607378",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:29:01.528517",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria. Firstly, the 'type' of edit is identical in both the predicted and ground truth edits, as both are categorized under 'punctuation'. Secondly, for the text content accuracy, both the ground truth and the predicted edit identify the core change required: adding a comma after 'howling', in 'howling over' to form 'howling, over'. The correction of inserting the comma is precisely captured in the predicted edit. Therefore, since both the edit type and the essential text content change are accurately aligned with the ground truth, the prediction is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit in both evaluated criteria: \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edits specify the type as 'punctuation', which is an exact match. Therefore, the edit type is accurate.\n\n2. Text Content Accuracy: The core change in the ground truth is adding a comma to change 'pockets and' to 'pockets, and'. The predicted edit also captures this change accurately, indicating the same transformation of the text by inserting a comma between 'pockets' and 'and'. Thus, the essential edit required by the ground truth is effectively captured in the prediction. \n\nSince both the edit type and the text content match accurately, the prediction is considered correct."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:06.963473",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:28:59.819068",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the edit type as 'capitalization', which matches the ground truth edit type. In terms of text content accuracy, the prediction captures the core change that the ground truth identifies, which is the capitalization of the word 'why'. The ground truth identifies the full original text as 'why did he dine' changing to 'Why did he dine', while the prediction only mentions the capitalization of 'why' to 'Why'. The essential change, which is the capitalization of the first word, is accurately captured in the prediction. Therefore, the predicted edit is correct according to both edit type accuracy and text content accuracy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "!",
"corrected_text": "?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the edit type as 'punctuation,' which matches the ground truth edit type. For text content accuracy, the essential change in the punctuation from '!' to '?' is accurately captured in the predicted edit. While the ground truth specifies the full phrase 'cold stones!' to 'cold stones?', the core change is in the punctuation mark. The predicted edit lists the specific change from '!' to '?', which correctly captures the intended punctuation alteration highlighted in the ground truth. Therefore, since both the edit type and the text content change are accurate, the prediction is deemed correct."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:10.874323",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:29:07.727161",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit based on the evaluation criteria. \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edit have the type 'capitalization', which exactly matches. Therefore, the edit type criterion is satisfied.\n\n2. Text Content Accuracy: The essential change in both the ground truth and predicted edit involves capitalizing the first letter of 'why'. The ground truth mentions 'why did he dine' being corrected to 'Why did he dine', while the prediction indicates 'why' corrected to 'Why'. The prediction focuses on the core change, which is the capitalization of 'why'. Including only 'why' as context does not impact the correctness since the core change of capitalizing 'why' is accurately captured.\n\nSince both criteria for edit type and text content are satisfied, the predicted edit is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "!",
"corrected_text": "?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The evaluation centers around two criteria: Edit Type Accuracy and Text Content Accuracy. \n\n1. Edit Type Accuracy: Both the ground truth and the predicted edit classify the change as a 'punctuation' edit. This means the edit type matches exactly, satisfying the first criterion.\n\n2. Text Content Accuracy: The core change identified by the ground truth is altering the punctuation from an exclamation mark '!' to a question mark '?'. The prediction focuses on this same change by identifying the original punctuation '!' and the corrected punctuation '?'. Although the ground truth includes some surrounding context ('cold stones!'), the essential change is the punctuation itself, which the prediction captures accurately.\n\nBoth criteria are fulfilled by the predicted edit, making it a correct representation of the ground truth edit."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:11.202585",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:29:30.499573",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "why",
"corrected_text": "\"why",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "!",
"corrected_text": "?",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:07.578735",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:29:30.500405",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit based on the provided evaluation criteria. \n\n1. **Edit Type Accuracy:** Both the ground truth and the predicted edit specify 'capitalization' as the type of edit. This matches exactly, fulfilling the first criterion. \n\n2. **Text Content Accuracy:** The essential change in both the ground truth and the predicted edit involves capitalizing 'if' to 'If.' Although the ground truth includes 'he' in the context, the core change focuses solely on the word 'if' being capitalized. The prediction captures this essential change correctly, as it identifies the lowercase 'if' and the corrected uppercase 'If.' Including or excluding 'he' from the context does not affect the core change, as the significant action is the capitalization of 'if.' \n\nTherefore, the predicted edit accurately matches the ground truth in both the edit type and the core text content change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. \n\n1. **Edit Type Accuracy**: Both the predicted edit and the ground truth edit specify the type as 'replacement', which matches exactly. Therefore, the edit type is accurate.\n\n2. **Text Content Accuracy**: The ground truth edit indicates a change from 'said Clennam' to 'asked Clennam', focusing on changing 'said' to 'asked'. The predicted edit specifically captures the change from 'said' to 'asked', which is the core change required. Although the predicted edit includes less context ('Clennam' is omitted), the essential change from 'said' to 'asked' is accurately captured. The prediction includes the correct transformation and therefore fulfills the ground truth's intention.\n\nOverall, since both criteria match the ground truth, the prediction is correct."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:15.493986",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:29:23.349529",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is evaluated on two primary criteria: 'Edit Type Accuracy' and 'Text Content Accuracy'.\n\n1. Edit Type Accuracy: The predicted edit and the ground truth edit both specify the type as 'capitalization', indicating that the intention is to change the capitalization of a word in the text. Since the edit type matches exactly, this criterion is satisfied.\n\n2. Text Content Accuracy: The ground truth edit specifies changing 'if he' to 'If he'. The predicted edit specifies changing 'if' to 'If'. The core change both texts are referring to is the capitalization of the initial 'i' in 'if'. The predicted edit captures this essential change. Although it does not explicitly reference the 'he', the primary intention of changing the capitalization is evident in both, and the context is preserved since 'if' is a part of 'if he'.\n\nOverall, both the edit type and text content accurately align with the ground truth\u2019s intention. Therefore, the predicted edit is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit correctly identifies the edit type as 'replacement', matching the ground truth. Since the types match exactly, the edit type criterion is satisfied. \n\n2. Text Content Accuracy: The core change in both the ground truth and the prediction involves replacing the word 'said' with 'asked'. The ground truth specifies 'said Clennam' is to be changed to 'asked Clennam', while the prediction specifies 'said' is to change to 'asked'. The essential edit, which is changing 'said' to 'asked', is captured by the prediction. Therefore, the prediction accurately captures the intention of the ground truth in terms of text content, even though it uses a shorter context.\n\nBoth criteria are addressed, and the prediction aligns with the ground truth in both type and content, thus the prediction is correct."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:10.783129",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:29:18.271500",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit and ground truth share the same edit type, 'replacement,' thus fulfilling the criterion for Edit Type Accuracy. For Text Content Accuracy, the core change in both cases is the replacement of the word 'said' with 'asked.' The ground truth mentions the entire phrase 'said Clennam,' while the prediction isolates the word 'said.' However, in alignment with the parameters, the essential change is captured as it highlights the same core alteration in the sentence. Therefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:48.114607",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:30:08.433415",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit based on the evaluation criteria. \n\n1. Edit Type Accuracy: Both the ground truth and predicted edit specify the edit type as 'punctuation'. Since the edit types match exactly, the prediction meets this criterion.\n\n2. Text Content Accuracy: The ground truth and predicted edits involve changing 'high road' to 'high-road'. This represents the core change involving a punctuation alteration (inserting a hyphen). The predicted edit captures this change correctly with additional context matching the ground truth.\n\nGiven that both the edit type and text content correctly match the ground truth's intention, the predicted edit is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both the edit type and text content. Firstly, the edit type 'punctuation' is identical in both the predicted and ground truth edits, satisfying the first criterion for edit type accuracy. Secondly, the text content change from 'materials costing' to 'materials, costing' is accurately captured in the predicted edit, which correctly includes the addition of a comma between 'materials' and 'costing'. There is no additional context affecting the core change, and the core change (the insertion of a comma) is represented accurately. Therefore, the prediction satisfies the requirements for both criteria."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. Regarding Edit Type Accuracy, both the ground truth and predicted edits have the type 'punctuation,' which matches exactly, satisfying the criterion for edit type matching. For Text Content Accuracy, the core change identified in both the ground truth and predicted edits is the insertion of a comma in 'about and' to make it 'about, and.' The prediction captures this essential core change accurately, even if it includes additional context. As both conditions for evaluation (type and content) have been met, the prediction is deemed correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit in both criteria: \n\n1. **Edit Type Accuracy**: Both the ground truth and the predicted edit have 'punctuation' as the type. This signifies that the nature of the change involves punctuation, thus satisfying the edit type accuracy requirement.\n\n2. **Text Content Accuracy**: The core change identified in both the ground truth and predicted edit is the addition of a comma between 'earnestly' and 'telling', changing 'earnestly telling' to 'earnestly, telling'. This correction is accurately reflected in the prediction, meeting the requirement for text content accuracy. \n\nTherefore, the prediction is correct as it accurately matches the essential change and type as identified in the ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly matches the ground truth edit in both criteria. \n\n1. Edit Type Accuracy: The type of the edit is 'punctuation' for both the ground truth and the prediction, indicating that the same kind of modification (addition or removal of punctuation) is being performed. Therefore, the edit type matches exactly. \n\n2. Text Content Accuracy: The core change identified by the ground truth is the addition of a comma between 'said' and 'when', turning 'said when' into 'said, when'. The predicted edit captures this same change, modifying the text from 'said when' to 'said, when'. Although additional context is allowed around the core change, in this case, both focus directly on the essential alteration. Hence, the text content accurately reflects the ground truth's intention. \n\nGiven that both the edit type and the text content align perfectly with the ground truth, the prediction is evaluated as correct."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria required for evaluation. Firstly, the 'edit type' is accurately marked as 'punctuation' in both the ground truth and predicted edits, thus satisfying the requirement for type accuracy. Secondly, the 'text content' shows the same correction: changing 'said' to 'said,'. This indicates the model has correctly captured the essential change as identified by the ground truth, which is the addition of a comma after the word 'said.' Since both the edit type and the core textual change align perfectly with the ground truth, the predicted edit is evaluated as correct."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit accurately in both criteria specified. \n\n1. **Edit Type Accuracy**: Both the ground truth and the predicted edit specify the type as 'punctuation'. This matches exactly, fulfilling the edit type requirement.\n\n2. **Text Content Accuracy**: The ground truth requires changing 'Clennam in' to 'Clennam, in'. The predicted edit also identifies this correction correctly, matching the core change by adding a comma after 'Clennam'. Therefore, the essential edit has been captured accurately.\n\nSince both the edit type and the core textual change align with the ground truth, the prediction is correct."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "associa-tion",
"corrected_text": "association",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:45.334922",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:30:08.201479",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit precisely matches the ground truth edit in both criteria required for evaluation. \n\n1. Edit Type Accuracy: The edit type is 'punctuation' for both the ground truth and the predicted edit. This shows that the predicted edit correctly identifies the type of change that was made, matching the ground truth exactly.\n\n2. Text Content Accuracy: The core textual change identified in both the ground truth and prediction is transforming 'high road' to 'high-road'. The prediction captures this essential alteration accurately. There are no discrepancies or additional contexts affecting the fundamental change required. \n\nAs both the edit type and the change in text content are correctly captured in the prediction, the evaluation concludes that the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is correct based on the evaluation criteria. Firstly, the edit type 'punctuation' is exactly the same in both the ground truth and predicted edit, meeting the edit type accuracy requirement. Secondly, for text content accuracy, the prediction captures the core change indicated by the ground truth: adding a comma between 'about' and 'and' in the text 'about, and'. Since both the type and the essential text change align perfectly with the ground truth, the prediction is considered accurate."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The evaluation focuses on two primary criteria: Edit Type Accuracy and Text Content Accuracy. \n\n1. Edit Type Accuracy: \nThe edit type in both the ground truth and the predicted edit is 'punctuation'. This matches exactly, so the prediction satisfies the Edit Type Accuracy criterion.\n\n2. Text Content Accuracy: \nThe ground truth specifies a change from 'earnestly telling' to 'earnestly, telling'. The predicted edit also reflects this same change, from 'earnestly telling' to 'earnestly, telling'. The core change, which is the addition of a comma after 'earnestly', is accurately captured by the prediction. The essential edit is present in both the ground truth and the prediction such that the text content matches the intention of the edit.\n\nSince both criteria are met, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria required for correctness. Firstly, the edit type in both the ground truth and prediction is 'punctuation', which means the type was correctly identified. Secondly, the text content accuracy criterion is met since the core change from 'said when' to 'said, when' is captured in the prediction exactly as it was in the ground truth. There are no additional context errors visible, and the key change, which is the addition of a comma after the word 'said', is correctly implemented. Therefore, the predicted edit is considered correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is correct because it matches the ground truth on both criteria. For edit type accuracy, both the ground truth and the predicted edit indicate a 'punctuation' change, which aligns perfectly. Regarding text content accuracy, both the ground truth and the prediction change 'said' to 'said,', which is the core edit identified. The essential change is the addition of a comma after 'said', and this is accurately captured in the prediction. Since both the edit type and the text content align with the ground truth intent, the prediction is accurate."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials, costing",
"corrected_text": "materials; costing",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:46.305553",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:30:14.915716",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth in both criteria evaluated. Firstly, the edit type is 'punctuation' in both the ground truth and the prediction, which is essential for type accuracy. Secondly, the text content accuracy is maintained as the core change from 'high road' to 'high-road' is correctly identified and executed in the prediction, precisely capturing the intended correction. Thus, the prediction accurately reflects the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately matches the ground truth edit based on both evaluation criteria. Firstly, the edit type is 'punctuation' in both the ground truth and the prediction, satisfying the requirement for edit type accuracy. Secondly, the text content change has been correctly captured: both the ground truth and the prediction change 'materials costing' to 'materials, costing', correctly indicating the addition of a comma between 'materials' and 'costing'. Therefore, the predicted edit correctly captures the intention of the ground truth edit in all respects considered for this evaluation."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is considered correct because it matches the ground truth in both of the required criteria. Firstly, the edit type is 'punctuation' in both the ground truth and the prediction, ensuring edit type accuracy. Secondly, the text content change from 'about and' to 'about, and' is accurately captured in the predicted edit. Even if additional context were included, this core change would remain correct. Therefore, the predicted edit fully meets the requirements of the ground truth edit, capturing both the type and the essential text change accurately."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit captures the intention of the ground truth edit with precision. Firstly, the edit type, 'punctuation,' matches exactly between the predicted and ground truth edits. No discrepancy exists in identifying the type of change intended. Secondly, the core textual change is accurately reflected in the predicted edit. Both the ground truth and the prediction show the transition from 'earnestly telling' to 'earnestly, telling,' indicating the addition of a comma after 'earnestly.' This demonstrates that the essential punctuation change, which is the insertion of a comma, has been correctly understood and implemented in the prediction. Thus, based on both criteria\u2014edit type accuracy and text content accuracy\u2014the predicted edit is correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the 'punctuation' type, which matches the ground truth type precisely. Regarding text content accuracy, the predicted edit accurately captures the core change specified in the ground truth, which is adding a comma after 'said.' Both the original and corrected texts in the prediction match the ground truth: 'said when' to 'said, when,' ensuring the essential edit is correctly captured. As both the edit type and the text content accurately reflect the ground truth's intention without any discrepancies, the prediction is considered correct."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria. Firstly, the edit type in both cases is 'punctuation', which is identical, meeting the Edit Type Accuracy requirement. Secondly, for Text Content Accuracy, the essential change from 'said' to 'said,' is accurately captured. The prediction must capture the core change of adding a comma after 'said', which it does, fulfilling the Text Content Accuracy criterion. Therefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit based on the criteria provided. Both the predicted and ground truth edits have the same edit type, 'punctuation,' which means the prediction matches the required edit type accuracy. In terms of text content accuracy, the prediction changes 'Clennam in' to 'Clennam, in,' which reflects the core change intended by the ground truth. The addition of the comma after 'Clennam' is correctly captured in both edits. Since both key criteria\u2014edit type and core text content\u2014are accurately matched in the prediction, the predicted edit is deemed correct."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "associa-tion",
"corrected_text": "association",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:31.983076",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:30:06.207903",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly matches the ground truth edit in both criteria. Firstly, the edit type is identified as 'punctuation' in both the ground truth and predicted edit, indicating that the model correctly identified the nature of the change. Secondly, the text content accuracy is satisfied as both the ground truth and predicted edit change the text from 'Sun and Shadow' to 'Sun and Shadow.', accurately capturing the insertion of the period punctuation. Since both the type and content match, the predicted edit is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:36.397378",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:30:08.206558",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "The predicted edit successfully captures the intention of the ground truth edit under the given evaluation criteria. \n\n1. Edit Type Accuracy: \n - The edit type in both the ground truth and the predicted edit is 'punctuation'. They match exactly, thus satisfying the edit type accuracy requirement.\n\n2. Text Content Accuracy: \n - The ground truth edit changes 'Sun and Shadow' to 'Sun and Shadow.', indicating the addition of a period at the end of the text.\n - The predicted edit also changes 'Sun and Shadow' to 'Sun and Shadow.' by adding the period at the end. This captures the essential change that the ground truth identifies, which is the punctuation addition.\n - While the line number differs, the provided criteria specify that line numbers should be ignored for this evaluation.\n\nGiven these points, the prediction accurately reflects the core change intended by the ground truth edit, resulting in a correct prediction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:35.898132",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:30:19.868733",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit according to both criteria specified. \n\n1. Edit Type Accuracy: The edit type in both the ground truth and the predicted edit is 'punctuation'. Therefore, according to the first criterion, the edit type is correctly matched.\n\n2. Text Content Accuracy: Both the ground truth and the predicted edit show a change from 'Sun and Shadow' to 'Sun and Shadow.', indicating an addition of a period at the end of the sentence. Since both specify the same core textual change, the text content is accurately captured by the prediction.\n\nOverall, the predicted edit correctly reflects the intention of the ground truth edit according to the specified evaluation criteria."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:46.956093",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:30:38.787849",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both the 'edit type' and 'text content' criteria. The edit type is a 'replacement' in both the ground truth and predicted edits, satisfying the Edit Type Accuracy requirement. Regarding Text Content Accuracy, the original text 'Saone' is correctly identified and replaced with the corrected text 'Sa\u00f4ne' in both the ground truth and the prediction. The CORE change, which is the addition of the circumflex accent on the 'o', is captured accurately in the prediction. Therefore, the prediction is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly reflects the ground truth in both criteria of evaluation. Firstly, the 'type' of edit in both the ground truth and the prediction is 'punctuation', indicating a precise match in terms of what the edit type is. Second, the 'corrected_text' field in both cases shows the same essential change: introducing a comma in 'Chalons was' to become 'Chalons, was'. Since the core alteration required in the text content is the same\u2014the insertion of a comma between 'Chalons' and 'was'\u2014the prediction perfectly captures the intention of the ground truth. Accordingly, both criteria for a correct edit are satisfied, thus making the prediction correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:49.288053",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:30:22.949346",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The evaluation of the predicted edit against the ground truth edit reveals that both key criteria are met. First, regarding 'Edit Type Accuracy', the predicted edit type is 'replacement', which matches exactly with the ground truth edit type. Therefore, the type of change has been correctly identified. Second, under 'Text Content Accuracy', the predicted correction involves changing 'Saone' to 'Sa\u00f4ne', which aligns perfectly with the core change identified in the ground truth. The essential character change, from 'o' to '\u00f4', is captured precisely, indicating that the predicted edit conveys the same intended modification as the ground truth edit. Since both the edit type and the core text content match the ground truth, the predicted edit accurately captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit and the ground truth edit both have the same type 'punctuation', which means the edit type matches exactly. Regarding the text content, the core change identified by the ground truth is the addition of a comma after 'Chalons'. The predicted edit also captures this essential punctuation change, although it leaves out the word 'was' from the original and corrected text context. The omission of 'was' is permissible as the essential edit was performed correctly: adding a comma after 'Chalons'. Therefore, the predicted edit accurately captures the ground truth's intention regarding the punctuation change."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:34:49.138434",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:30:24.351458",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria. First, the edit type is 'replacement' in both the predicted edit and the ground truth, which ensures edit type accuracy. Second, the text content accuracy is maintained as the core change from 'Saone' to 'Sa\u00f4ne' is identical in both the ground truth and predicted edits. There is no issue with additional context since both edits focus on the exact text change required. Therefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:04.526022",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:31:05.525964",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. Both the edit type and the text content are accurately matched. \n\n1. Edit Type Accuracy: Both the ground truth and predicted edit types are 'punctuation', which is a direct match. \n\n2. Text Content Accuracy: The ground truth shows the original text 'heart so' being corrected to 'heart, so', and the prediction reflects the same correction by adding a comma. The core change, which is the insertion of a comma after 'heart', is captured perfectly in the prediction. Therefore, the edit type and content both align correctly with the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit perfectly in terms of both criteria. The edit type is 'punctuation' in both the ground truth and the prediction, satisfying the edit type accuracy. Additionally, the text content accuracy is met because both the ground truth and predicted edits involve the same correction, changing 'length said' to 'length, said'. There are no additional unnecessary changes or discrepancies in the core change, indicating that the predicted edit accurately captures the essential edit represented in the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria evaluated. Firstly, the edit type is 'replacement' in both cases, ensuring perfect edit type accuracy. Secondly, both the original text ('I hope so') and the corrected text ('I hoped so') match exactly between the ground truth and the prediction, satisfying the text content accuracy requirement. The prediction accurately reflects the core change intended by the ground truth. Hence, the predicted edit is correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from",
"corrected_text": "thanked him indeed from",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:10.157770",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:31:29.004224",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length",
"corrected_text": "length,",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The edit type in both the ground truth and the predicted edit is 'punctuation', which matches exactly. Therefore, the prediction is correct in terms of edit type.\n\n2. Text Content Accuracy: The core change in the ground truth is the addition of a comma to the word 'length', resulting in 'length, said'. The predicted edit also correctly captures this key change by transforming 'length' to 'length,'. Although the prediction does not specify the subsequent word 'said', the essential punctuation change is correctly portrayed. According to the instructions, inclusion of additional context is acceptable as long as the core change is correctly identified, which is the case here.\n\nThus, both criteria are satisfied, and the prediction is accurate as per the ground truth's intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in terms of both edit type and core change. Firstly, the edit type 'replacement' is consistent in both the ground truth and the prediction, satisfying the first criterion of edit type accuracy. Secondly, in terms of text content accuracy, the essential change in both the ground truth and predicted edits is the alteration of 'hope' to 'hoped'. Despite the ground truth specifying a longer string ('I hope so' to 'I hoped so'), the core change captured by the prediction from 'hope' to 'hoped' is accurate and captures the essential grammatical correction. Therefore, the prediction correctly reflects the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ";",
"corrected_text": ",",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart",
"corrected_text": "heart,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:06.163792",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:31:34.585815",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both aspects required for evaluation. \n\n1. **Edit Type Accuracy**: The edit type in both the ground truth and the predicted edit is 'punctuation.' Since they match exactly, the edit type is correct.\n\n2. **Text Content Accuracy**: The essential change specified in the ground truth is the insertion of a comma, changing 'length said' to 'length, said.' The predicted edit correctly captures this change without any deviation. Although both the original and corrected text segments provided in the prediction are exactly the same as those in the ground truth, this does not affect the evaluation as the core change (inserting a comma after 'length') is correctly reflected.\n\nTherefore, both criteria are satisfied, making the prediction correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Both the ground truth edit and the predicted edit have identified the same type of change, a 'replacement'. This satisfies the Edit Type Accuracy criterion as the 'replacement' type matches exactly in both instances. For Text Content Accuracy, the ground truth specifies changing 'I hope so' to 'I hoped so', whereas the predicted edit changes 'hope' to 'hoped'. Here, the core change from 'hope' to 'hoped' is correctly identified and executed in the predicted edit. The prediction includes less context, but since additional context is permissible, the essential edit is still captured accurately. Therefore, the predicted edit aligns with the intention of the ground truth edit in both criteria, making it correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart ;",
"corrected_text": "heart;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "heart",
"corrected_text": "heart indeed",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:12.461448",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:31:18.092496",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately matches the ground truth edit in both criteria required for evaluation. First, the Edit Type Accuracy is met because both the ground truth and the predicted edits identify the type as 'punctuation'. This shows that the predicted edit correctly identified the category of change being made to the text. Second, the Text Content Accuracy is achieved because the prediction captures the core change specified in the ground truth: the insertion of a comma after 'hers'. Both the original and corrected text in the prediction are identical to those in the ground truth, meaning the essential edit is the same. Thus, the prediction successfully reflects the intention of the ground truth edit, justifying a correct evaluation."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit. Firstly, the edit type is 'punctuation' for both the ground truth and the predicted edit, which matches exactly. Secondly, the text content accuracy criterion is satisfied: both the ground truth and prediction present the change of adding a comma in the phrase 'hers matters', transforming it into 'hers, matters'. There are no unnecessary additions or omissions in the prediction that alter the core change intended by the ground truth. Therefore, the prediction is correct in both edit type and text content."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit on both criteria. Firstly, the edit type is 'capitalization' in both the ground truth and the prediction, indicating that the prediction correctly identifies the nature of the correction as a change in capitalization. Secondly, the text content accuracy criterion is met as both the original text 'they' and the corrected text 'They' are identical in both the ground truth and the predicted edit. This demonstrates that the core change, specifically capitalizing the initial 'T' in 'they', is correctly captured in the prediction, fulfilling the requirement for text content accuracy. Therefore, the prediction is correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment,",
"corrected_text": "moment",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:19.742287",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:31:28.809390",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The edit type 'punctuation' matches exactly between the ground truth edit and the predicted edit. Therefore, the edit type is correct. \n\n2. Text Content Accuracy: The core change in both the ground truth and predicted edit is the insertion of a comma after 'moment'. The ground truth specifies 'moment with' to 'moment, with', while the predicted edit specifies 'moment' to 'moment,'. The predicted edit accurately captures the essential change, which is the insertion of a comma after 'moment'. The additional context ('with') included in the ground truth does not alter the core change being captured correctly in the prediction.\n\nSince both criteria are met, the predicted edit is deemed correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately matches the ground truth edit based on the evaluation criteria. Firstly, the edit type is 'capitalization' for both the ground truth and the prediction, satisfying the requirement for Edit Type Accuracy. Secondly, concerning Text Content Accuracy, both the original text and the corrected text in the prediction match those in the ground truth ('they' to 'They'). Thus, the core change identified by the ground truth is correctly captured by the prediction. The prediction does not include unnecessary additional context and focuses solely on the essential change required as per the ground truth. Therefore, the predicted edit is correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hesitation,",
"corrected_text": "hesitation",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hers",
"corrected_text": "her",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "her",
"corrected_text": "her,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:19.474782",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:31:21.508969",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly matches the ground truth edit in terms of both edit type and text content. \n\n1. **Edit Type Accuracy**: The edit type in both the ground truth and predicted edit is 'punctuation,' which matches exactly, so the edit type is correct.\n\n2. **Text Content Accuracy**: The core change in both the ground truth and prediction is from 'hers and' to 'hers, and.' The essential correction of introducing a comma after 'hers' is accurately captured in the prediction. Additionally, there is no discrepancy in the context provided, and the essential change is correctly identified.\n\nSince both criteria are fulfilled, the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit is correct because it matches the ground truth edit in both the edit type and text content. The edit type is 'punctuation' in both cases, indicating that the focus is on modifying the punctuation in the text. The original text and corrected text in the prediction ('hers matters' to 'hers, matters') are identical to the ground truth, effectively capturing the core change that was intended by the ground truth. Since the essential edit is correctly captured without any discrepancies in the edit type or the core text content, the prediction accurately reflects the intention of the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures both the edit type and the text content change specified by the ground truth. Firstly, the edit type 'punctuation' matches exactly, fulfilling the requirement for edit type accuracy. Secondly, the text content change is precisely captured; both the original text 'moment with' and the corrected text 'moment, with' are identical between the ground truth and the prediction. This meets the criterion of text content accuracy, which focuses on capturing the core change indicated by the ground truth. Therefore, both criteria are satisfied, and the predicted edit is correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both required aspects. First, the edit type is 'punctuation' in both the ground truth and the predicted edit. Second, the text content accuracy is achieved because the essential change, which is the insertion of a comma resulting in 'beautiful' changing to 'beautiful,', is correctly captured. Both edits reflect the same modification, so the prediction is correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both the edit type and text content. Both the ground truth and predicted edits are of type 'capitalization', which is the correct edit type. Additionally, the text change from 'they' to 'They' is correctly captured in both the ground truth and prediction. Since the core change\u2014the capitalization of the word 'they' to 'They'\u2014has been accurately identified and the edit type is correctly noted as 'capitalization', the prediction is considered correct according to the evaluation criteria."
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:13.171525",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:31:41.198307",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria. Firstly, the edit type is 'punctuation' in both the ground truth and the predicted edit, indicating that the type of change is correctly identified. Secondly, the text content accuracy is also maintained, as the essential edit is the insertion of a comma in the phrase 'shore and' to become 'shore, and'. The prediction captures this core change accurately. Therefore, the predicted edit is correct based on the criteria provided."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both key aspects evaluated: edit type accuracy and text content accuracy. 1. Edit Type Accuracy: Both the ground truth and predicted edit types are 'replacement', which is an exact match. 2. Text Content Accuracy: The predicted edit accurately captures the core change identified in the ground truth, replacing 'that' with 'than'. There is no deviation in the actual change required. Therefore, the prediction is correct as it satisfies both criteria perfectly."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:22.435601",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:31:36.875800",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you\u201d",
"corrected_text": "Thank you,\u201d",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The edit type for both the ground truth and predicted edit is 'punctuation'. Thus, the edit type is correctly matched.\n\n2. Text Content Accuracy: The core change in the ground truth is the replacement of the exclamation mark '!' with a comma ',' at the end of 'Thank you'. The predicted edit captures the same punctuation change, although it involves an extra character, the closing double quotation mark. This discrepancy is merely an expansion of context and does not alter the core intention, which is the punctuation change (from '!' to ','). Since the core change is correctly captured, and extra context before/after the change is permissible, the text content is considered accurate in matching the ground truth's intention.\n\nTherefore, the prediction correctly identifies both the type and nature of the edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit has the type 'punctuation,' which matches exactly with the ground truth type 'punctuation.' Therefore, the edit type is accurate. \n\n2. Text Content Accuracy: The core change indicated by the ground truth is the addition of a comma after 'shore' (changing 'shore and' to 'shore, and'). The predicted edit captures this essential punctuation change by altering 'shore' to 'shore,'. Although the context (additional words) differs, the key transformation of adding a comma after 'shore' is correctly captured. Therefore, the text content accurately reflects the ground truth's intention.\n\nBoth criteria are met, so the prediction is correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly matches the ground truth in both key evaluation aspects: Edit Type Accuracy and Text Content Accuracy. Firstly, the edit type 'replacement' corresponds exactly between the predicted and ground truth edits, satisfying the criterion for Edit Type Accuracy. Secondly, the text content shows an identical replacement from 'that' to 'than,' which is precisely what the ground truth specifies. Therefore, the core change intended by the ground truth has been accurately captured by the predicted edit. No discrepancy in context is present, and thus the prediction fulfills both criteria effectively."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "that,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy",
"corrected_text": "fancy,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:25.203434",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:32:24.505267",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit accurately based on the evaluation criteria. \n\n1. Edit Type Accuracy: The edit type in both the ground truth and the predicted edit is 'replacement', which matches exactly, fulfilling the required condition for edit type accuracy. \n\n2. Text Content Accuracy: The core change identified in both the ground truth and the prediction is the replacement of 'that' with 'than'. This change is captured precisely in both, fulfilling the criteria for text content accuracy. There are no issues with additional context because the essential component of the edit is captured correctly.\n\nSince both criteria are met, the prediction is deemed correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you, murmured",
"corrected_text": "you,\u201d murmured",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "in our fancy",
"corrected_text": "",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank",
"corrected_text": "\u201cThank",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ",",
"corrected_text": "",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:53.325292",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:32:36.025843",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "MRS. FLINTWINCH GOES ON DREAMING",
"corrected_text": "MRS. FLINTWINCH GOES ON DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is 'punctuation', which matches exactly with the ground truth edit type. Therefore, there is no discrepancy here.\n\n2. Text Content Accuracy: The ground truth specifies a change to the word 'DREAMING' to add a period, resulting in 'DREAMING.'. The predicted edit expands the context to 'MRS. FLINTWINCH GOES ON DREAMING' and applies the same punctuation change, transforming it to 'MRS. FLINTWINCH GOES ON DREAMING.'. The essential edit, which is adding a period after 'DREAMING', is accurately captured and applied in the prediction. Therefore, despite the inclusion of additional context, the core change required by the ground truth is correctly executed.\n\nBoth criteria are satisfied; hence, the prediction is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both the type and the text content criteria. In terms of edit type accuracy, both the ground truth and the prediction have the edit type 'capitalization', which matches exactly. Regarding text content accuracy, the core change required by the ground truth is changing 'city' to 'City', and this change is exactly what the prediction captures as well. The corrected text in both the ground truth and prediction is the capitalization of 'City' from 'city'. There are no additional or omitted words that affect the core change. Therefore, the prediction accurately reflects the intention behind the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures all aspects of the ground truth edit. \n\n1. **Edit Type Accuracy**: Both the ground truth and the predicted edit specify the edit type as 'punctuation', which is an exact match.\n\n2. **Text Content Accuracy**: The change in text from 'them,' to 'them;' is correctly captured in both the ground truth and the predicted edit. The core change, which is the replacement of the comma with a semicolon, is identical in both the predicted and the ground truth edits. The contextual text 'them,' -> 'them;' is matched without any discrepancies.\n\nSince both criteria meet the requirements accurately, the prediction is considered correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. Firstly, the edit type 'punctuation' matches exactly with the ground truth edit type, satisfying the Edit Type Accuracy criterion. Secondly, the text content accuracy is met as the predicted edit identifies the same core change from 'it,' to 'it;' as the ground truth. Both edits reflect the exact punctuation change without any discrepancies. Thus, since both the type and content are accurately captured, the predicted edit is considered correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit perfectly in both the type and the text content. The edit type is 'punctuation' in both the ground truth and prediction, which satisfies the requirement for edit type accuracy. Furthermore, the text content change from 'stand-still,' to 'stand-still;' exactly corresponds in the prediction to that in the ground truth. Both the original and corrected texts are identical, capturing the essential edit accurately without any discrepancies. Thus, the prediction captures the intention of the ground truth completely and accurately on both required criteria."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in all required aspects. Both the edit type and the text content are accurately captured. \n\n1. Edit Type Accuracy: The type in both the ground truth and prediction is 'punctuation', which matches exactly. Therefore, the edit type criterion is satisfied. \n\n2. Text Content Accuracy: The change in the text from 'existence,' to 'existence;' is correctly identified in the predicted edit. Although additional context is acceptable, the core change involving the punctuation from a comma to a semicolon is accurately captured in the prediction. \n\nSince both criteria are met, the prediction is correct according to the ground truth's intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "seen",
"corrected_text": "seen;",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "night, each",
"corrected_text": "night, to each",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,",
"corrected_text": "view;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liege",
"corrected_text": "liege lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:46.999156",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:32:42.787643",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit based on both specified criteria. \n\n1. **Edit Type Accuracy**: Both the ground truth and predicted edits have the same edit type, 'capitalization'. This means the predicted edit correctly identifies the type of correction intended by the ground truth.\n\n2. **Text Content Accuracy**: The core change identified by the ground truth is from 'city' to 'City'. The predicted edit captures this change precisely. There are no discrepancies in the text content or in the nature of the change, fulfilling the requirement that the prediction should reflect the essential change presented in the ground truth.\n\nGiven that both criteria are met adequately, the predicted edit is considered correct in capturing the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. Firstly, the edit type is 'punctuation', which matches exactly between the ground truth and the prediction, satisfying the Edit Type Accuracy criterion. Secondly, the Text Content Accuracy criterion is met as the core change identified in the ground truth is changing 'them,' to 'them;', and this exact change is reflected in the predicted edit as well. Both original and corrected text segments match perfectly with the ground truth. Therefore, the predicted edit is accurate according to the evaluation criteria."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both criteria: Edit Type Accuracy and Text Content Accuracy. For Edit Type Accuracy, both the ground truth and the predicted edit specify 'punctuation' as the type, which is correct and matches exactly. For Text Content Accuracy, the essential change is from 'it,' to 'it;', which is captured accurately in the predicted edit. Since both criteria are met with the same core change and edit type, the prediction is considered correct."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit perfectly matches the ground truth edit in both required aspects. First, the edit type is 'punctuation' in both the ground truth and the predicted edit, satisfying the criterion for Edit Type Accuracy. Second, the Text Content Accuracy is upheld as well, as the predicted edit captures the core change from 'stand-still,' to 'stand-still;'. Therefore, the essential punctuation change from a comma to a semicolon is correctly identified in the predicted edit, accurately reflecting the intention of the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly captures the intention of the ground truth edit. \n\n1. Edit Type Accuracy: \n - Both the ground truth and the predicted edit types are 'punctuation', which means the edit types match exactly. \n\n2. Text Content Accuracy: \n - The essential change in the ground truth is the addition of a comma after 'view' transforming it from 'view' to 'view,'. \n - The predicted edit also captures this essential change by showing the correction from 'view by' to 'view, by'. \n - Although the predicted edit includes additional context ('by'), this does not affect the evaluation as the core change, which is the addition of the comma after 'view', is correctly identified. \n - Therefore, the text content accuracy criterion is satisfied, as the essential change from 'view' to 'view,' is accurately captured.\n\nSince both criteria are met, the edit is considered correct."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both critical aspects required for accuracy evaluation. Firstly, the 'edit type' in both the ground truth and prediction is 'punctuation', which means there is no discrepancy in the type of change identified. Secondly, the 'text content accuracy' is upheld because the core change from 'existence,' to 'existence;' is accurately captured by the predicted edit. The essential punctuation change is correctly identified and applied, indicating that the prediction meets the ground truth's indicated intention effectively."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "xxix",
"corrected_text": "xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless; when",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:52.724651",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:33:06.337343",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly matches the ground truth edit in both criteria evaluated: \n\n1. Edit Type Accuracy: The edit type in both the ground truth and predicted edit is 'capitalization.' This matches exactly, indicating the predicted edit correctly identifies the nature of the change.\n\n2. Text Content Accuracy: The core change identified by the ground truth is modifying 'city' to 'City,' reflecting a capitalization adjustment. The predicted edit mirrors this change accurately, capturing the essential transformation of the text. Since the prediction doesn't include superfluous or incorrect text, it successfully represents the intended edit.\n\nBoth the edit type and text content are in perfect alignment with the ground truth, thereby confirming the prediction is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately captures the intention of the ground truth edit for the following reasons: \n\n1. Edit Type Accuracy: Both the ground truth and predicted edits are categorized under the same edit type, which is 'punctuation'. This means that the model has successfully identified the type of editorial change required, thus satisfying the criterion for edit type accuracy.\n\n2. Text Content Accuracy: The original text in both the ground truth and predicted edits is 'it,', and the corrected text is 'it;', indicating a change of punctuation from a comma to a semicolon after the word 'it'. This reflects the core change that the ground truth specifies. The prediction covers the exact change seen in the ground truth without any discrepancies in the actual content that was altered.\n\nOverall, since both the edit type and the specific textual correction align perfectly with the ground truth, the prediction is considered correct."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth in both criteria. First, the 'edit type' is accurately identified as 'punctuation' in both the predicted and the ground truth edits. Second, the 'text content' shows the core change of altering the punctuation from a comma to a semicolon after 'stand-still'. Both texts correctly reflect this change without altering the essential meaning or structure of the correction. Therefore, based on the criteria of matching edit type and capturing the core text change, the predicted edit is accurate."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth in terms of edit type and captures the core change. \n\n1. Edit Type Accuracy: Both the ground truth and predicted edits are of type 'punctuation', indicating that the prediction's edit type is accurate.\n\n2. Text Content Accuracy: The ground truth correction involves adding a comma to the word 'view', changing it to 'view,'. The predicted correction takes 'view by' and changes it to 'view, by', which includes the same crucial punctuation change of adding a comma to 'view'. While the prediction includes an additional word 'by', it still correctly identifies and implements the core edit: the addition of a comma after 'view'. Therefore, the prediction accurately captures the intended correction from the ground truth. \n\nOverall, both criteria are satisfied, so the edit is correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in both the edit type and the text content. The edit type is 'punctuation', which is exactly the same as the ground truth. For text content accuracy, the change from 'existence,' to 'existence;' is identical in both the predicted and ground truth edits. Therefore, both the edit type and the essential textual change have been correctly captured by the prediction."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each recurring",
"corrected_text": "to each recurring",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:44.849780",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:32:49.047666",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam,",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that\u2014boiling-over",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:47.144344",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:32:39.851009",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "bubbling out of him",
"corrected_text": "bubbling out of him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit matches the ground truth edit in terms of edit type and text content accuracy. \n\n1. Edit Type Accuracy: Both the ground truth and predicted edits are of type 'italicize', indicating that the action to be performed on the text is the same.\n\n2. Text Content Accuracy: The ground truth specifies italicizing 'him', whereas the prediction involves italicizing a larger segment, 'bubbling out of him'. The essential change of italicizing 'him' is present in the prediction because 'him' is included within the broader context of 'bubbling out of him'. Therefore, the core change of italicizing the word 'him' is accurately captured, as per the ground truth intention.\n\nGiven these assessments, the predicted edit accurately reflects the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit accurately matches the ground truth edit in two critical areas: \n\n1. **Edit Type Accuracy**: Both the ground truth and the predicted edit are of type 'replacement'. This indicates that the model correctly understood the nature of the editorial change needed.\n\n2. **Text Content Accuracy**: The specific change required was from 'authorize' to 'authorise'. The predicted edit reflects this exact core change without any deviation. Even if there were more surrounding text, the core text content change aligns exactly with the ground truth information.\n\nGiven these evaluations\u2014despite any line number consideration, which has been explicitly disregarded here\u2014the predicted edit is entirely correct in both changing the spelling to British English and identifying the action needed as a 'replacement'."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ";",
"corrected_text": ",",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:45.470708",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:32:43.154720",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "The predicted edit correctly identifies the type of edit as 'replacement,' which matches the ground truth's specified edit type. Additionally, the text content change from 'authorize' to 'authorise' is accurately reflected in both the ground truth and the prediction. Hence, both the edit type accuracy and the text content accuracy criteria are met. The inclusion or exclusion of additional context is not relevant for this evaluation as the core change is accurately captured, leading to the conclusion that the prediction is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ";",
"corrected_text": ",",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that-boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of the driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:41.686909",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:32:56.277425",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "word I",
"corrected_text": "word, I",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "never",
"corrected_text": "never saw",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ", not",
"corrected_text": ",\u2014not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed-room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:55.365693",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:32:56.412980",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "For",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me,",
"corrected_text": "me;",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_openai_gpt_5_2",
"date": "2026-02-22T13:35:56.442481",
"annotator": "GPT-5.2 (OR)",
"annotation_date": "2026-02-22T13:33:43.398289",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "an",
"corrected_text": "wear",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "For ",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ",",
"corrected_text": ";",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2025-12-11",
"release_source": "https://openai.com/index/introducing-gpt-5-2/"
},
{
"model_name": "GPT-5.3 Chat",
"model_id": "gpt-5.3-chat",
"date": "2026-03-05T19:27:00.346596",
"shots": 2,
"config": {
"model_id": "gpt-5.3-chat",
"display_name": "GPT-5.3 Chat",
"shots": 2,
"temperature": 0.0,
"date": "2026-03-05",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:12.782669",
"file_results": [
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:22:36.701715",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:12.782669",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change adds a period to the end of the phrase \"Fellow Travellers\" (i.e., after \"Travellers\"). The predicted edit adds a period after \"Travellers\" as well, just with less surrounding context (it omits \"Fellow\"). Since the essential correction\u2014appending a period after \"Travellers\"\u2014is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the types match exactly. Text Content Accuracy: Both ground truth and prediction change \"yonder to-day\" to \"yonder, to-day\", inserting a comma after \"yonder\". The core punctuation change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same corrected text (\"pockets, and\"), capturing the identical core change\u2014adding a comma after \"pockets\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:22:42.287707",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:11.036168",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"Fellow Travellers\" to \"Fellow Travellers.\"\u2014i.e., it adds a period at the end of the phrase. The predicted edit changes \"Travellers\" to \"Travellers.\", which still captures the essential intended change: adding a terminal period after the final word. Although the prediction uses a narrower span (only the last word rather than the full phrase), it applies the same core punctuation correction at the same location (the end of the phrase)."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 14,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change the phrase \"pockets and\" to \"pockets, and\" by inserting a comma after \"pockets.\" This captures the core intended punctuation correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day Sir",
"corrected_text": "yonder, to-day, Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:22:43.062148",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:11.653064",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period at the end of the phrase \"Fellow Travellers\" (i.e., after \"Travellers\"). The prediction also adds a period after \"Travellers\". Although the prediction uses less surrounding context (it edits only \"Travellers\" rather than \"Fellow Travellers\"), it captures the same essential punctuation change at the same word boundary. Therefore, the core intended correction is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the identical core change: insertion of a comma after \"yonder\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to make it \"pockets, and\". The original and corrected texts match exactly, so the intended punctuation correction is captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:22:34.090984",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:21.954408",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "think, why",
"corrected_text": "think: why",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "stones",
"corrected_text": "stones!\u201d",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:22:52.068127",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:13.156391",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "think why",
"corrected_text": "think, why",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones\" She",
"corrected_text": "stones!\" She",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:22:52.066843",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:09.654181",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "think, why",
"corrected_text": "think: why",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!\" She",
"corrected_text": "stones!\" \u2014 She",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:23:14.900435",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:10.701731",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 33,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"if he\" to \"If he\", which is specifically capitalizing the word \"if\" at the start of the phrase. The prediction changes \"if\" to \"If\", capturing the same core capitalization change. Although it omits the following word (\"he\"), the essential intended correction (capitalizing \"if\") is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 36,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit, so the edit type matches.\n\n2) Text Content Accuracy: The ground truth replaces \"said Clennam\" with \"asked Clennam\". The prediction replaces \"said\" with \"asked\". This captures the core intended change (\"said\" \u2192 \"asked\"). Although the prediction omits the surrounding word \"Clennam\" from both original and corrected text, it still represents the same essential replacement within the phrase, so the intended correction is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "do",
"corrected_text": "do?",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:23:06.584467",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:15.606886",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "do",
"corrected_text": "do?",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "do?",
"corrected_text": "do? If",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:23:08.885876",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:16.759221",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "do",
"corrected_text": "do?",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:23:10.706200",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:53.867746",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The predicted change matches the ground truth precisely, correcting \"high road\" to \"high-road\" by inserting a hyphen. The original and corrected text are identical to the ground truth, so the core intended edit is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Labouring in this sea as all barks labour in cross seas",
"corrected_text": "Labouring in this sea, as all barks labour in cross seas",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:23:38.834948",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:55.298518",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"high road\") and the same correction (\"high-road\"), capturing the identical core change of inserting a hyphen. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 11,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same core textual change: inserting a comma after \"about\" in the phrase \"about and\" to make it \"about, and\". The original_text and corrected_text match the ground truth\u2019s intention exactly.\n\nLine numbers are ignored as instructed, so the differing line_number does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original phrase (\"earnestly telling\") and the same correction (\"earnestly, telling\"), inserting a comma after \"earnestly.\" This captures the core intended change with no discrepancy in the edited text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"said\" in the phrase \"said when\" \u2192 \"said, when.\" The predicted edit \"said\" \u2192 \"said,\" captures the same core punctuation insertion (adding a comma immediately after \"said\"). Although it omits the following word \"when\" as context, it still represents the essential intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 26,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2. Text Content Accuracy: Both edits change \"Clennam in\" to \"Clennam, in\" by inserting a comma after \"Clennam.\" This captures the core intended correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:23:34.946666",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:03.635864",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: The ground truth changes \"high road\" to \"high-road\" (inserting a hyphen). The prediction specifies the same original text and the same corrected text, capturing the identical core change. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same transformation from \"materials costing\" to \"materials, costing\", correctly capturing the intended insertion of a comma after \"materials\". The core change is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the intended insertion of a comma after \"earnestly\". Line numbers are ignored, and no other differences affect the core edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 26,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The prediction also inserts a comma after \"Clennam\" (\"Clennam in amazement\" \u2192 \"Clennam, in amazement\"). The added words \"amazement\" are extra context and do not alter the core punctuation change. Therefore, the prediction captures the intended edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Labouring in this sea as all barks labour in cross seas",
"corrected_text": "Labouring in this sea, as all barks labour in cross seas",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:23:35.564378",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:54.287929",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have the type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. The core change is identical, so the predicted edit matches the ground truth intention."
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:23:45.135707",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:54.303770",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the core intended correction. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:23:55.674574",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:18:55.759554",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. The core intended punctuation change is captured identically."
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:24:13.085500",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:04.604376",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"dull autumn\") and the same corrected text (\"dull, autumn\"), capturing the identical core change\u2014adding a comma after \"dull\". Line numbers are ignored as instructed. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change \"Chalons was\" to \"Chalons, was\" by inserting a comma after \"Chalons\". This captures the same core punctuation correction with identical original and corrected text. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with",
"corrected_text": "soddened with",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"sodden\" to \"soddened\". The prediction changes \"sodden with\" to \"soddened with\", which includes additional surrounding context (the word \"with\") but preserves the same core substitution: \"sodden\" \u2192 \"soddened\". This aligns with the ground truth\u2019s intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone it",
"corrected_text": "Saone, it",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:24:12.162597",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:05.258687",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text \"dull autumn\" and the same correction \"dull, autumn\", capturing the intended insertion of a comma after \"dull\". Ignoring line numbers, the essential change is identical."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both edits target the same original string (\"Chalons was\") and apply the same correction (\"Chalons, was\"), inserting a comma after \"Chalons.\" This captures the core intended change precisely. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sodden with wet",
"corrected_text": "soddened with wet",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is the word-level replacement \"sodden\" \u2192 \"soddened\". The prediction applies the same core substitution while including extra surrounding context (\"with wet\") on both sides: \"sodden with wet\" \u2192 \"soddened with wet\". Since the essential modification (changing \"sodden\" to \"soddened\") is preserved, the text content accurately captures the intended edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone it",
"corrected_text": "Saone, it",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:24:25.045155",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:07.589515",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"dull autumn\") and the same corrected text (\"dull, autumn\"), capturing the core intended change: inserting a comma after \"dull\". Line numbers are ignored as instructed. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change\u2014adding a comma after \"Chalons.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone it",
"corrected_text": "Saone, it",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "soddenwith",
"corrected_text": "sodden with",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:24:20.761924",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:35.283233",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart",
"corrected_text": "indeed thanked him from her heart",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "I hoped so",
"corrected_text": "I hoped so",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:24:35.088836",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:28.003703",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart",
"corrected_text": "thanked him indeed from her heart",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said Clennam",
"corrected_text": "at length, said Clennam",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:24:35.469998",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:24.894442",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart",
"corrected_text": "indeed thanked him from her heart",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "I hoped so;",
"corrected_text": "I hoped so",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:24:47.603845",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:35.455069",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they are to be married.",
"corrected_text": "They are to be married.",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:24:54.127740",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:35.647342",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they are to be married.",
"corrected_text": "They are to be married.",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:24:59.169988",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:46.294590",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 19,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly \u201cThey",
"corrected_text": "directly: \u201cThey",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:25:06.733394",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:45.500093",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "whatever",
"corrected_text": "",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you",
"corrected_text": "\"Thank you\"",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:25:26.088306",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:48.993486",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "in our fancy",
"corrected_text": "",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you\u201d",
"corrected_text": "Thank you,\u201d",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:25:28.425252",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:19:51.090390",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "within-doors",
"corrected_text": "within doors",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "whatever",
"corrected_text": "",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:25:44.748413",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:20:12.522518",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Chapter XXIX\") and the same corrected text (\"Chapter XXIX.\"), capturing the identical core change of adding a period. Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 12,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the predicted edit are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change, adding a comma immediately after \"motionless\" within a slightly larger span (\"motionless when\" \u2192 \"motionless, when\"). The added context does not alter the essential correction and accurately captures the intended punctuation change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.09999999999999998,
"fp": 0.45,
"fn": 0.45,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 13,
"line_diff": 3,
"line_number_penalty": 0.9,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change within a larger context (\"view by\" \u2192 \"view, by\"), which correctly captures the intended punctuation insertion after \"view.\" The additional context does not alter the essential edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them;",
"corrected_text": "them,",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it;",
"corrected_text": "it:",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still; to",
"corrected_text": "stand-still, to",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence; is",
"corrected_text": "existence, is",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "city",
"corrected_text": "city (so)",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "each",
"corrected_text": "to each",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,at",
"corrected_text": "existence, at",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:26:40.655092",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:20:12.261062",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\". The original_text and corrected_text match exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit expands the span to include the following word (\"existence at\" \u2192 \"existence, at\"), but it still captures the same essential correction: inserting a comma right after \"existence\". The added context does not alter the core intended change.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "her liege lord",
"corrected_text": "her liege-lord",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "city",
"corrected_text": "city (car)",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "return of",
"corrected_text": "return to",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them images",
"corrected_text": "them; images",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view; by",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:26:32.450715",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:20:14.232823",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Chapter XXIX\") and the same corrected text (\"Chapter XXIX.\"), capturing the identical core change of adding a period. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence at the hour",
"corrected_text": "existence, at the hour",
"observed_line_number": 12,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change within a longer span of text (\"existence at the hour\" \u2192 \"existence, at the hour\"), which correctly captures the essential intent: inserting a comma immediately after \"existence.\" Additional surrounding context is acceptable and does not alter the core edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 15,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit expands the context to include the following word (\"view by\" \u2192 \"view, by\"), but the core change is the same: inserting a comma immediately after \"view.\" This accurately captures the intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 16,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to replace the punctuation immediately following \u201cexistence\u201d from a comma to a semicolon (\u201cexistence,\u201d \u2192 \u201cexistence;\u201d). The predicted edit results in \u201cexistence;\u201d appearing before the following word (\u201cexistence; is\u201d), which captures the core intended punctuation outcome: a semicolon after \u201cexistence\u201d instead of the prior punctuation. Although the predicted original span (\u201cexistence is\u201d) omits the comma shown in the ground truth original (\u201cexistence,\u201d), the essential correction\u2014placing a semicolon after \u201cexistence\u201d\u2014is correctly represented."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "her liege lord",
"corrected_text": "her liege-lord",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "return of the same sequences",
"corrected_text": "return to the same sequences",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to suppose",
"corrected_text": "it; to suppose",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:25:58.590064",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:20:10.372018",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it,\u201d said Clennam:",
"corrected_text": "it,\u201d said Clennam;",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes the punctuation after \"Clennam\" from a colon to a semicolon (\"Clennam:\" \u2192 \"Clennam;\"). The prediction makes the same core change, but includes additional surrounding context (\"it,\u201d said Clennam:\" \u2192 \"it,\u201d said Clennam;\"). This still captures the essential intended correction (colon to semicolon after \"Clennam\").\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "a little premature",
"corrected_text": "a little premature.",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Your own recompense for all this, Mr. Pancks,",
"corrected_text": "Your own recompense for all this, Mr. Pancks,",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "driest",
"corrected_text": "driest of snorts",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:26:22.402239",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:20:21.355844",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam:",
"corrected_text": "said Clennam;",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is converting the punctuation after \"Clennam\" from a colon to a semicolon (\"Clennam:\" \u2192 \"Clennam;\"). The prediction captures the same core change, only with additional surrounding context (\"said Clennam:\" \u2192 \"said Clennam;\"). Since the essential punctuation substitution at \"Clennam\" is identical, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "been a little premature",
"corrected_text": "been a little premature.",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:26:30.681279",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:20:26.754812",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said Clennam:",
"corrected_text": "said Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest of snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:26:48.119475",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:20:23.401169",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 10,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same change from \"quickly that\" to \"quickly, that\" (inserting a comma after \"quickly\"). This captures the core intended correction precisely. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:26:56.053383",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:20:29.382035",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"For Mr. Pancks\" to \"For, Mr. Pancks,\" inserting a comma after \"For\". The core punctuation change is identical in both."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.3-chat",
"date": "2026-03-05T19:27:00.346596",
"annotator": "GPT-5.3 Chat",
"annotation_date": "2026-03-05T19:20:37.564361",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"For Mr. Pancks\") and the same correction (inserting a comma after \"For\" to make \"For, Mr. Pancks\"). The core punctuation change is identical. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2026-03-03",
"release_source": "https://openai.com/index/gpt-5-3-instant/",
"release_notes": "Alias to gpt-5.3-chat-latest; using the GPT-5.3 Instant launch date."
},
{
"model_name": "GPT-5.3 Codex (OR)",
"model_id": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:32:52.748648",
"shots": 2,
"config": {
"model_id": "or_openai_gpt_5_3_codex",
"display_name": "GPT-5.3 Codex (OR)",
"shots": 2,
"temperature": 0.0,
"date": "2026-03-05",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:38.003520",
"file_results": [
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:28:14.133277",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:38.003520",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The core intended punctuation change is captured exactly with the same original and corrected text. Therefore, the prediction correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"pockets and\" to \"pockets, and,\" correctly capturing the intended insertion of a comma after \"pockets.\" Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:28:23.335100",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:35.122672",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: adding a period to \"Fellow Travellers\" to make it \"Fellow Travellers.\". The original_text and corrected_text match the ground truth\u2019s core intended correction exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n\nText Content Accuracy: Both specify the same core change from \"Sir\" to \"sir\". The original and corrected texts match the ground truth\u2019s intended capitalization change with no discrepancy.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both specify the same change, inserting a comma after \"pockets\" (\"pockets and\" \u2192 \"pockets, and\"). The core intended punctuation correction is captured exactly with no discrepancy."
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:28:13.807197",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:45.498326",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: Both ground truth and prediction change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"capitalization,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change from \"Sir\" to \"sir\". There is no discrepancy in the original or corrected text, so the essential intended edit is captured correctly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both specify the same change: inserting a comma after \"pockets\" so that \"pockets and\" becomes \"pockets, and\". The core punctuation correction is identical in original and corrected text."
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:28:21.732036",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:55.436286",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial word of the phrase, changing \"why did he dine\" to \"Why did he dine.\" The prediction captures the same core change by converting \"why\" to \"Why.\" Although it provides less surrounding context, it correctly identifies and applies the essential capitalization correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!\"",
"corrected_text": "stones?\"",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:28:43.626477",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:47.053871",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the first word of the phrase, changing \"why did he dine\" to \"Why did he dine.\" The prediction captures this core change by changing \"why\" to \"Why.\" Although it includes less surrounding context, it correctly represents the essential capitalization correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation mark to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction captures the same core change on the relevant word/segment: \"stones!\" \u2192 \"stones?\". Although it includes less surrounding context (omitting \"cold\"), it still reflects the essential intended edit (replacing ! with ? after \"stones\"). Therefore, the text content matches the ground truth's intention."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:28:44.235290",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:55.946389",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"No sir\") and the same correction (\"No, sir\"), inserting a comma after \"No\". This captures the core intended change precisely with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to capitalize the initial \"w\" in the phrase \"why did he dine\" \u2192 \"Why did he dine.\" The prediction captures this core intended change by editing \"why\" \u2192 \"Why.\" Although it includes less surrounding context, it still reflects the essential capitalization correction at the start of the sentence."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "think, why",
"corrected_text": "think, \"Why",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:28:51.822373",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:28.387727",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial word in the phrase \"if he\" \u2192 \"If he.\" The prediction captures the same essential change by capitalizing \"if\" \u2192 \"If.\" Although it includes less surrounding context (omitting \"he\"), it still reflects the core correction (capitalizing the leading \"i\" in \"if\"), so the intended edit is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces the word \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the same core substitution \"said\" \u2192 \"asked\". Although it omits the following context (\"Clennam\"), it still represents the essential intended change. Therefore, the text content matches the ground truth's intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "do if",
"corrected_text": "do? if",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:28:53.496823",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:27.718068",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth changes \"if he\" to \"If he,\" i.e., capitalizing the initial \"i\" in \"if\" while leaving \"he\" unchanged. The prediction changes \"if\" to \"If,\" capturing the same core correction (capitalizing \"if\"). Although it includes less surrounding context (omits \"he\"), it still represents the essential intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces \"said Clennam\" with \"asked Clennam\", meaning the core change is \"said\" \u2192 \"asked\" while retaining the surrounding context (\"Clennam\"). The predicted edit replaces \"said\" with \"asked\" without including the trailing context, but it captures the essential intended substitution. Since additional surrounding words are optional and the core change is correctly identified, the text content matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:29:11.630310",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:21:37.149509",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase \"if he\" \u2192 \"If he.\" The prediction captures the same core change by converting \"if\" \u2192 \"If.\" Although it omits the following word (\"he\") from the span, the essential intended correction\u2014capitalizing \"if\"\u2014is correctly represented, and additional surrounding context is not required."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the core intended substitution \"said\" \u2192 \"asked\". Although it omits the surrounding context word (\"Clennam\"), capturing the essential word-level replacement is sufficient and aligns with the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:29:23.932423",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:23:09.023420",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both the ground truth and the predicted edit are labeled as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same change from \"high road\" to \"high-road,\" correctly capturing the intended hyphenation. Therefore, the prediction matches the ground truth in both type and core text change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth. Text Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing\" by inserting a comma after \"materials.\" The core punctuation change is identical, so the prediction matches the intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"about\" in the phrase \"about and\" to make it \"about, and\". The original and corrected text strings match the ground truth intention exactly."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: removing the comma in \"association, did\" to produce \"association did.\" This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), inserting a comma after \"earnestly.\" The core change is identical, so the content matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after the word \"said\" in the phrase \"said when\" (i.e., \"said when\" \u2192 \"said, when\"). The predicted edit \"said\" \u2192 \"said,\" captures the same essential punctuation change (adding a comma immediately after \"said\"). Although the prediction omits the following word \"when\" as context, it still reflects the core intended correction."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in amazement",
"corrected_text": "Clennam, in amazement",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The prediction performs the same core punctuation change\u2014adding a comma after \"Clennam\"\u2014while including additional surrounding context (\"in amazement\"). This preserves the essential intended edit."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said, when",
"corrected_text": "said when",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:29:27.256077",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:23:03.633225",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly. Text Content Accuracy: Both ground truth and prediction change \"high road\" to \"high-road,\" capturing the identical core correction (inserting a hyphen). Therefore, the prediction correctly matches the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation: \"materials costing\" \u2192 \"materials, costing,\" inserting a comma after \"materials.\" This captures the core intended punctuation change with identical original and corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"about and\" to \"about, and\", correctly capturing the insertion of a comma after \"about\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and predicted edit are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly.\" The core intended punctuation change is captured exactly with no discrepancy."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same change, inserting a comma after \"said\" to convert \"said when\" to \"said, when.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same core change: replacing \"said\" with \"said,\" by adding a comma. There is no discrepancy in the original or corrected text. Therefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have the type set to \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same corrected text (\"Clennam, in\"), capturing the identical core change\u2014adding a comma after \"Clennam\". Therefore the prediction matches the ground truth\u2019s intended correction."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:29:35.146980",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:23:38.825768",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"high road\" to \"high-road,\" capturing the same core punctuation change (inserting a hyphen). There is no discrepancy in the original or corrected text content.\n\nTherefore, the prediction correctly captures the intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing\" by inserting a comma after \"materials.\" The core punctuation correction is identical. Therefore, the predicted edit correctly captures the intended change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change, inserting a comma to convert \"about and\" to \"about, and.\" The essential edit is identical."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same change from \"association, did\" to \"association did\", i.e., removing the comma after \"association\". This captures the core intended correction with identical original and corrected text."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), inserting a comma after \"earnestly.\" This captures the identical core change with no discrepancy."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after the word \"said\" in the phrase \"said when\" \u2192 \"said, when\". The prediction captures the same essential change by editing \"said\" \u2192 \"said,\". Although it does not include the following word \"when\" in its context, it still represents the core intended punctuation insertion at the correct location (after \"said\"). Therefore, the text content matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same correction (\"Clennam, in\"), capturing the identical core change\u2014adding a comma after \"Clennam\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said, when",
"corrected_text": "said when",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:29:35.500612",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:22:31.817099",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as type \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. This captures the same core correction with identical original and corrected text. Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:29:51.151848",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:22:36.549334",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. The core punctuation change is identical, so the intended correction is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:29:53.312553",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:22:43.593395",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2. Text Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. The core punctuation change is identical in the predicted edit, matching the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:30:07.805443",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:23:35.338157",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"dull autumn\" to \"dull, autumn\" by inserting a comma after \"dull\". The core punctuation change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Saone\") and the same corrected text (\"Sa\u00f4ne\"), capturing the intended addition of the circumflex accent over the 'o'. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). This captures the same core correction with identical original and corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change\u2014adding a comma after \"Chalons\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XI",
"corrected_text": "XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE,",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:30:07.655109",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:23:10.977767",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"dull autumn\") and the same corrected text (\"dull, autumn\"), capturing the identical core change of inserting a comma after \"dull\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both indicate the same core change: replacing \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The original and corrected text strings are identical between ground truth and prediction. Therefore the predicted edit captures the intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly. Text Content Accuracy: Both edits replace the original text \"Saone\" with the corrected text \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core change is identical in the prediction and the ground truth. Therefore, the predicted edit correctly captures the intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"Chalons was\" to \"Chalons, was\", inserting a comma after \"Chalons\". The core intended change is identical in the prediction and ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:30:29.350551",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:23:39.197801",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"dull autumn\") and the same corrected text (\"dull, autumn\"), capturing the identical core change\u2014adding a comma after \"dull\". Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same substitution from \"Saone\" to \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core intended correction is captured identically. Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core change is identical, with no discrepancy in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"man slowly\" to \"man, slowly\", correctly capturing the intended insertion of a comma after \"man\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change\u2014adding a comma after \"Chalons\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE,",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:30:26.007403",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:24:31.387744",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"heart\"\u2014with identical original_text (\"heart so\") and corrected_text (\"heart, so\"). Therefore, the prediction correctly captures the intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction performs the same core punctuation insertion in the same place, but includes extra surrounding context (\"at length said\" \u2192 \"at length, said\"). Including additional context is allowed as long as the essential change is the same, which it is here.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "(as she did, if it be the source of earnestness),",
"corrected_text": "indeed,",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:30:32.951319",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:24:13.984457",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction performs the same core change\u2014adding a comma after \"length\"\u2014but includes extra surrounding context (\"at length said\" \u2192 \"at length, said\"). This preserves the essential punctuation correction identified in the ground truth.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the phrase \"I hope so\" to \"I hoped so,\" which is essentially changing the word \"hope\" to \"hoped\" while leaving the surrounding words unchanged. The prediction replaces \"hope\" with \"hoped,\" capturing the core intended correction (present tense to past tense). Although the prediction includes less surrounding context than the ground truth, it correctly represents the essential change."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him from her heart",
"corrected_text": "indeed thanked him from her heart",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:30:36.547464",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:25:11.564456",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"heart so\") and the same corrected text (\"heart, so\"), capturing the core intended change of inserting a comma after \"heart\". Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction performs the same core change, just with extra surrounding context (\"at length said\" \u2192 \"at length, said\"). Since the essential punctuation insertion is identical and correctly placed, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"). The core change\u2014shifting from present tense \"hope\" to past tense \"hoped\"\u2014is captured identically. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "did",
"corrected_text": "did indeed",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:30:58.951378",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:25:08.562866",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"hers\" in the phrase \"hers and\" to become \"hers, and.\" The original and corrected text strings match the ground truth\u2019s intended punctuation change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers\". The core punctuation change is identical in the prediction and the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"moment with\" to \"moment, with\", inserting a comma after \"moment\". The core intended punctuation change is captured precisely with identical original and corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "directly, They",
"corrected_text": "directly, \u201cThey",
"observed_line_number": 32,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:31:02.479970",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:24:19.541973",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"hers and\" to \"hers, and,\" inserting a comma after \"hers.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers\". The core change is identical in the prediction and ground truth. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the identical correction from \"moment with\" to \"moment, with,\" inserting a comma after \"moment.\" The core change is captured exactly with no discrepancy."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same corrected text (\"beautiful, that\"), capturing the identical core change\u2014adding a comma after \"beautiful\". Therefore the prediction matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth. Text Content Accuracy: Both ground truth and prediction change the same token from \"they\" to \"They,\" capturing the identical core correction with no discrepancy. Therefore, the prediction correctly matches the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:31:07.545745",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:24:53.001572",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change the phrase from \"hers and\" to \"hers, and\", inserting a comma after \"hers\". This captures the exact core punctuation correction intended by the ground truth with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction apply the same core change: inserting a comma after \"hers\" in the phrase \"hers matters,\" resulting in \"hers, matters.\" The original and corrected text fields match exactly, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change\u2014adding a comma after \"moment\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same corrected text (\"beautiful, that\"), capturing the identical core change of inserting a comma after \"beautiful\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same core change, converting \"they\" to \"They\". There is no discrepancy in original or corrected text. Therefore, the prediction captures the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:31:07.431742",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:25:28.358531",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore, the predicted edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing the original text \"that\" with the corrected text \"than\". This captures the same core change with no discrepancy. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:31:34.853341",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:25:25.284752",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"shore and\" to \"shore, and\" by inserting a comma after \"shore\". The original_text and corrected_text are identical in both, capturing the same core change with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth specifies a \"replacement\" and the predicted edit is also a \"replacement\", so the type matches exactly.\nText Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\". This captures the same core change with no discrepancy. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you\"",
"corrected_text": "Thank you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:31:36.152323",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:25:11.001654",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit specify the type as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits change the phrase from \"shore and\" to \"shore, and\", inserting a comma after \"shore\". This captures the core intended punctuation change with identical original and corrected text. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as a \"replacement\", so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing the original text \"that\" with the corrected text \"than\". The core change is identical with no discrepancy. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "\"Thank you\"",
"corrected_text": "\"Thank you,\"",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:32:05.028305",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:26:25.954830",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: Both specify the same transformation: \"city\" \u2192 \"City.\" This captures the core intended change with no discrepancy.\n\nTherefore, the predicted edit correctly matches the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing the original text \"of\" with the corrected text \"to\". This captures the exact core change required. Line numbers are ignored as instructed and do not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth indicates inserting a comma immediately after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change, adding a comma after \"existence\" while including additional surrounding context (\"existence at\" \u2192 \"existence, at\"). This preserves the essential intended punctuation change.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction inserts that same comma after \"motionless\" while including the following word as context (\"motionless when\" \u2192 \"motionless, when\"). This captures the core intended punctuation change accurately."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma immediately after \"view\" within a larger phrase (\"view by\" \u2192 \"view, by\"). This captures the same core punctuation insertion at the correct location, with only additional surrounding context included. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them images",
"corrected_text": "them; images",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:31:41.563871",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:26:06.977317",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"city\" to \"City\", capturing the identical core change (capitalizing the first letter). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n2. Text Content Accuracy: Both specify replacing the original text \"of\" with the corrected text \"to\". The core intended change is identical, with no discrepancy in the word being replaced or the replacement word."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: replacing the comma with a semicolon (\"them,\" \u2192 \"them;\"). The original and corrected texts are identical between ground truth and prediction, so the intended punctuation correction is captured correctly."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same core correction, changing \"it,\" to \"it;\". There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text \"stand-still,\" and the same correction to \"stand-still;\", capturing the identical core change (comma to semicolon). Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change within a larger context (\"view by\" \u2192 \"view, by\"), which still places the comma immediately after \"view.\" This captures the essential intended punctuation edit accurately."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change in the same location: \"existence,\" is corrected to \"existence;\". The core intended punctuation change (comma to semicolon) is captured exactly."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "lord and",
"corrected_text": "lord, and",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:32:21.211683",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:26:21.594799",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the edit type matches exactly. Text Content Accuracy: Both specify changing \"city\" to \"City,\" which captures the same core correction (capitalizing the initial letter). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly. Text Content Accuracy: Both edits replace the original text \"of\" with the corrected text \"to\". The core change is identical, with no discrepancy in the replaced or replacement text. Therefore, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"them,\") and the same corrected text (\"them;\"), capturing the identical core change from a comma to a semicolon after \"them\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it, to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"it,\" to \"it;\" (comma to semicolon after \"it\"). The prediction changes \"it, to\" to \"it; to,\" which includes extra surrounding context but preserves the same core punctuation change at the same point (comma after \"it\" becomes a semicolon). Therefore, it captures the essential intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still, to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation immediately following \"stand-still\" from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The predicted edit applies the same core change, but includes additional surrounding context (\"stand-still, to\" \u2192 \"stand-still; to\"). The essential punctuation change is identical and correctly captured."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit applies the same essential change within a slightly larger context (\"view by\" \u2192 \"view, by\"), which still places the comma immediately after \"view.\" This captures the core intended punctuation correction."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence, is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing the comma after \"existence\" with a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction applies the same core punctuation change while including additional surrounding context (\"existence, is\" \u2192 \"existence; is\"). This preserves the essential intended correction.\n\nBecause both the edit type and the core text change match the ground truth intention, the predicted edit is correct."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence, at",
"corrected_text": "existence; at",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless; when",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:32:10.325491",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:27:03.282354",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both indicate the same core change, replacing \"authorize\" with \"authorise\". There is no discrepancy in the original or corrected text. Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "Your own recompense for all this, Mr. Pancks,",
"corrected_text": "*Your own recompense for all this, Mr. Pancks,*",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that-boiling-over",
"corrected_text": "that boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "him)",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:32:14.630846",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:27:28.444918",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "*him*",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"italicize,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth indicates that the word \"him\" should be italicized without changing the literal characters (original_text and corrected_text both \"him\"). The prediction represents the italic formatting explicitly as \"*him*\" in corrected_text. Although this adds markup characters, it accurately captures the core intended change: applying italics to the word \"him\". The essential edit (italics applied to \"him\") is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the same original text \"authorize\" with the same corrected text \"authorise\". This captures the identical core change (American to British spelling). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:32:40.983774",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:26:51.052463",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly. Text Content Accuracy: Both identify the same target text \"him\" with no change in wording (only formatting/italicization), matching the core intended edit. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"authorize\") and the same corrected text (\"authorise\"), capturing the identical core spelling change. Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that-boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:32:42.544329",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:26:59.627239",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have the type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". The core change is identical. Therefore, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "ForMr.",
"corrected_text": "For Mr.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:32:49.346929",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:26:50.135081",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "ForMr.",
"corrected_text": "For Mr.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "quickly,that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me, not",
"corrected_text": "me; not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_openai_gpt_5_3_codex",
"date": "2026-03-05T19:32:52.748648",
"annotator": "GPT-5.3 Codex (OR)",
"annotation_date": "2026-03-05T19:26:41.139696",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". The core punctuation change is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "ForMr.",
"corrected_text": "For Mr.",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2026-02-05",
"release_source": "https://openai.com/index/introducing-gpt-5-3-codex/",
"release_notes": "OpenRouter route to the same GPT-5.3-Codex release."
},
{
"model_name": "GPT-5.4",
"model_id": "gpt-5.4",
"date": "2026-03-05T19:46:34.815396",
"shots": 2,
"config": {
"model_id": "gpt-5.4",
"display_name": "GPT-5.4",
"shots": 2,
"temperature": 0.0,
"date": "2026-03-05",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:27:38.114334",
"file_results": [
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:34:07.339845",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:27:38.114334",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period to the end of the phrase \"Fellow Travellers\" (i.e., making it \"Fellow Travellers.\"). The predicted edit adds a period after \"Travellers\" (\"Travellers\" \u2192 \"Travellers.\"). Even though it omits the preceding word \"Fellow\" from the original/corrected span, it captures the core intended change: inserting a period at the end of the same phrase/sentence ending. This is an acceptable narrower-context span that still reflects the essential correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the core intended change\u2014adding a comma after \"howling\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"Sir\" to \"sir\", capturing the identical core change (lowercasing the initial letter). Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2. Text Content Accuracy: Both specify the same transformation from \"pockets and\" to \"pockets, and\", correctly capturing the intended insertion of a comma after \"pockets\". No discrepancy in the core change."
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:34:10.251087",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:27:41.010484",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. This captures the same core punctuation correction with identical original and corrected text. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling",
"corrected_text": "howling,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"howling\" in the phrase \"howling over\" \u2192 \"howling, over\". The predicted edit captures this same core punctuation change by editing \"howling\" \u2192 \"howling,\". Although it does not include the following word \"over\" in its span, it still reflects the essential intended correction (adding a comma after \"howling\")."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder",
"corrected_text": "yonder,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth intends to insert a comma after the word \"yonder\" in the phrase \"yonder to-day\" \u2192 \"yonder, to-day\". The prediction captures this same core change by editing \"yonder\" \u2192 \"yonder,\". Although it omits the following word \"to-day\" as context, it still correctly represents the essential punctuation insertion at the correct location."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change from \"Sir\" to \"sir\". The predicted edit captures the identical original and corrected text, matching the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets",
"corrected_text": "pockets,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after the word \"pockets\" in the phrase \"pockets and\" \u2192 \"pockets, and\". The predicted edit captures the same core change by changing \"pockets\" \u2192 \"pockets,\". Although it omits the following word \"and\" from the context, it still represents the essential punctuation insertion at the correct location. Therefore, the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:34:07.274856",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:28:28.227294",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Fellow Travellers\") and the same corrected text (\"Fellow Travellers.\"), capturing the core intended change of adding a period at the end. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The ground truth change inserts a comma after \"howling\" (\"howling over\" \u2192 \"howling, over\"). The predicted edit specifies the same original text and the same corrected text, capturing the identical punctuation change. Therefore, the core intended edit is matched."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), inserting a comma after \"yonder\". The core change is captured identically."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"Sir\" to \"sir\". The core intended correction (lowercasing the word) is captured identically."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"pockets\" so that \"pockets and\" becomes \"pockets, and\". The original_text and corrected_text match the ground truth\u2019s core change exactly."
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:34:06.705619",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:28:17.591637",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"No sir\" to \"No, sir\", inserting a comma after \"No\". The core intended punctuation correction is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial word in the phrase \"why did he dine\" (changing \"why\" to \"Why\"). The predicted edit targets only the word \"why\" \u2192 \"Why\", which captures the essential correction (capitalizing the first word). Although it includes less surrounding context than the ground truth, it implements the same core change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation point to a question mark (\"cold stones!\" \u2192 \"cold stones?\"). The prediction captures the same core change on the same word/ending (\"stones!\" \u2192 \"stones?\"). Although it omits the preceding word \"cold\" in the span, it still represents the essential intended edit: replacing \"!\" with \"?\" at the end of the phrase. Therefore the text content matches the ground truth's intention."
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:34:31.466447",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:29:23.763067",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both apply the identical core punctuation change, converting \"No sir\" to \"No, sir\" by inserting a comma after \"No.\" There is no discrepancy in original or corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction identify the change as a \"capitalization\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase, changing \"why did he dine\" to \"Why did he dine\". The prediction focuses on just the changed token, \"why\" \u2192 \"Why\", which captures the core intended correction (capitalizing the first word). Although it omits the surrounding context (\"did he dine\"), it still accurately represents the essential edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing the exclamation mark with a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction captures the same core punctuation change on the relevant word/segment: \"stones!\" \u2192 \"stones?\". Although it omits the preceding word \"cold\", it still reflects the essential intended edit (changing ! to ? at the end of the phrase)."
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:34:34.736724",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:28:19.195396",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase (\"why did he dine\" \u2192 \"Why did he dine\"). The prediction targets the same core change by converting \"why\" \u2192 \"Why.\" Although it includes less surrounding context, it captures the essential intended edit: capitalizing the initial \"w\" in \"why.\""
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change, replacing the exclamation mark with a question mark: \"cold stones!\" \u2192 \"cold stones?\". The essential edit is captured precisely."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:34:31.138032",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:27:54.365998",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"if he\" to \"If he,\" which consists of capitalizing the initial \"if\" while leaving \"he\" unchanged. The prediction changes \"if\" to \"If,\" capturing the same core capitalization change, just with less surrounding context. Since the essential edit (lowercase to uppercase on \"if\") is correctly represented, the text content matches the ground truth's intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction replaces \"said\" \u2192 \"asked\" without including the trailing context \"Clennam\", but it captures the core intended change (the word substitution). Omitting surrounding context does not alter the essential correction being made."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:34:41.670263",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:28:17.077081",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"capitalization,\" which matches the ground truth exactly.\n\nText Content Accuracy: The ground truth capitalizes the word \"if\" in the phrase \"if he\" \u2192 \"If he.\" The prediction captures the same essential change by capitalizing \"if\" \u2192 \"If.\" Although it includes less surrounding context (omits \"he\"), it still represents the core intended correction (capitalizing the initial \"if\"). Therefore, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction replaces \"said\" \u2192 \"asked\" without including the following context (\"Clennam\"), but it captures the essential intended change (changing the verb from \"said\" to \"asked\"). Including less surrounding text is acceptable as long as the core change is correct.\n\nTherefore, both criteria are satisfied."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:34:57.528569",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:28:34.442682",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as \"capitalization\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase (\"if he\" \u2192 \"If he\"). The predicted edit capitalizes the same word but with less surrounding context (\"if\" \u2192 \"If\"). This still captures the core intended change\u2014capitalizing \"if\" to \"If\"\u2014and omitting the following word \"he\" does not alter the essential correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the word \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the essential substitution \"said\" \u2192 \"asked\" even though it omits the surrounding context (\"Clennam\"). Since the core intended change is the same word replacement, the text content matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:35:02.381285",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:30:19.642573",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"high road\" to \"high-road,\" capturing the same core hyphenation correction with identical original and corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing,\" inserting a comma after \"materials.\" This captures the core intended correction with identical original and corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same correction (\"about, and\"), capturing the identical core change: inserting a comma after \"about\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the types match exactly.\n\nText Content Accuracy: Both ground truth and prediction apply the same punctuation change, inserting a comma after \"earnestly\" in the phrase \"earnestly telling\" to become \"earnestly, telling\". The original_text and corrected_text match the ground truth\u2019s core change exactly."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same transformation, inserting a comma after \"said\" (\"said when\" \u2192 \"said, when\"). The core intended punctuation change is captured with identical original and corrected text."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core punctuation change, converting \"said\" to \"said,\" by adding a comma. The essential edit is identical, so the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"Clennam in\" to \"Clennam, in\", inserting a comma after \"Clennam\". This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:35:13.744341",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:30:01.181837",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The ground truth change is to hyphenate \"high road\" to \"high-road.\" The prediction identifies the same original text and the same corrected text, capturing the exact intended punctuation change.\n\nBecause both the edit type and the core text change match exactly, the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing\" by inserting a comma after \"materials.\" The core intended punctuation correction is captured exactly with no deviation."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"about and\" to \"about, and\", correctly inserting a comma after \"about\". The essential edit is identical."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both ground truth and prediction change \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly\". The core punctuation change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"said when\" to \"said, when\" by inserting a comma after \"said.\" The core punctuation change is identical with no discrepancy in original or corrected text."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, adding a comma after \"said\" (\"said\" \u2192 \"said,\"). The predicted edit fully captures the ground truth intention with no discrepancy."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have the type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same correction (\"Clennam, in\"), capturing the core intended change: inserting a comma after \"Clennam\". No discrepancy in the essential edit."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "associa- tion",
"corrected_text": "association",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:35:16.892096",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:31:08.762450",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The predicted change matches the ground truth core correction, converting \"high road\" to \"high-road\" by inserting a hyphen. There are no differences in the original or corrected text strings. Therefore, the prediction captures the intended edit precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"materials costing\" to \"materials, costing\" by inserting a comma after \"materials.\" This captures the same core punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"about\" to change \"about and\" to \"about, and.\" The essential punctuation correction is captured identically."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly\". The core punctuation change is identical, so the content matches the intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"said when\") and the same corrected text (\"said, when\"), capturing the core change of inserting a comma after \"said.\" Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core change: \"said\" \u2192 \"said,\" (adding a comma after \"said\"). The corrected text and original text match the ground truth\u2019s intention exactly."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"Clennam in\" to \"Clennam, in\", inserting a comma after \"Clennam\". This captures the same core punctuation correction with identical original and corrected text. Therefore, the content matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "harbour",
"corrected_text": "Harbour",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 44,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Vecchia, So",
"corrected_text": "Vecchia. So",
"observed_line_number": 46,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:35:21.661568",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:28:49.829447",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit have the type \"punctuation\", so the edit type matches exactly. Text Content Accuracy: Both specify the same change, adding a period to transform \"Sun and Shadow\" into \"Sun and Shadow.\". The core punctuation correction is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:35:28.238075",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:29:04.563966",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change\u2014adding a period to the end of \"Sun and Shadow\" to make it \"Sun and Shadow.\" The original and corrected texts match the ground truth\u2019s intention with no discrepancies."
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:35:46.926400",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:29:10.999967",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intention is to ensure the phrase ends with a period: \"Sun and Shadow\" \u2192 \"Sun and Shadow.\". The predicted edit also results in the same corrected text, changing the ending punctuation to a period: \"Sun and Shadow,\" \u2192 \"Sun and Shadow.\". Although the predicted original text includes a comma that is not present in the ground truth original, the core intended correction (making the ending punctuation a period) is captured correctly."
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:35:55.760730",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:30:51.762755",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"dull autumn\" \u2192 \"dull, autumn\", inserting a comma after \"dull\". The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core change is identical, with no discrepancies in original or corrected text. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex accent over the 'o'). The original and corrected texts are identical between ground truth and prediction, so the core change is captured precisely."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"man slowly\" \u2192 \"man, slowly\" (inserting a comma after \"man\"). This captures the core intended correction with no discrepancy."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same correction (\"Chalons, was\"), capturing the identical core change\u2014adding a comma after \"Chalons\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:36:06.821515",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:31:19.783547",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The core intended punctuation change is captured precisely with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"dull autumn\" to \"dull, autumn\", inserting a comma after \"dull\". The core punctuation correction is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both change the original text \"Saone\" to the corrected text \"Sa\u00f4ne\" (adding the circumflex over the o). The core intended correction is identical in both."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly. Text Content Accuracy: Both edits replace the same original text \"Saone\" with the same corrected text \"Sa\u00f4ne\" (adding the circumflex over the 'o'). This captures the core intended change precisely, with no discrepancy in the before/after text."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"man\" in the phrase \"man slowly\" to become \"man, slowly\". The original and corrected texts match the ground truth exactly. Therefore, the predicted edit captures the intended correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change: insertion of a comma after \"Chalons\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:36:03.051299",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:31:10.608333",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original phrase (\"man slowly\") and the same correction (\"man, slowly\"), capturing the identical core change\u2014adding a comma after \"man\". Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both ground truth and prediction change \"Chalons was\" to \"Chalons, was\", inserting a comma after \"Chalons\". The core punctuation change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sa\u00f4ne it",
"corrected_text": "Sa\u00f4ne, it",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:36:18.183431",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:32:13.027904",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits make the same core change, inserting a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The original and corrected text strings match the ground truth exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original_text (\"length said\") and the same corrected_text (\"length, said\"), capturing the identical core change\u2014adding a comma after \"length\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes the verb tense in the phrase \"I hope so\" \u2192 \"I hoped so\" (i.e., \"hope\" \u2192 \"hoped\" while leaving surrounding words intact). The prediction captures this same core change by replacing \"hope\" with \"hoped\". Although it provides less surrounding context than the ground truth, it correctly represents the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "she thanked",
"corrected_text": "she indeed thanked",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:36:26.172387",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:32:08.403613",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"heart so\" to \"heart, so,\" inserting a comma after \"heart.\" The core punctuation correction is identical."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 32,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits change the same original text \"length said\" to the same corrected text \"length, said\", inserting a comma after \"length\". This captures the core intended change precisely. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 35,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth replaces \"I hope so\" with \"I hoped so,\" which is essentially changing the verb form \"hope\" \u2192 \"hoped\" while keeping the surrounding words the same. The predicted edit captures this core change directly by replacing \"hope\" with \"hoped.\" Although it provides less surrounding context than the ground truth, it correctly represents the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked",
"corrected_text": "indeed thanked",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:36:30.096508",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:32:27.242875",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"heart so\") and the same corrected text (\"heart, so\"), capturing the core intended change of inserting a comma after \"heart\". No discrepancies in the essential edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction inserts the same comma at the same location, but includes extra surrounding context (\"at\" before). Since the core change\u2014adding a comma after \"length\" before \"said\"\u2014is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same substitution, changing \"I hope so\" to \"I hoped so.\" The core intended change (tense shift from present to past) is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked him",
"corrected_text": "indeed thanked him",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:36:38.595642",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:32:08.155449",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have the type set to \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"hers and\" \u2192 \"hers, and\", correctly capturing the intended insertion of a comma after \"hers\". There is no discrepancy in the core change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers\". The core punctuation change is identical, so the text content matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 24,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the types match exactly.\n\nText Content Accuracy: Both edits target the same original string \"moment with\" and correct it to \"moment, with\" by inserting a comma after \"moment\". This captures the core intended change precisely. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 26,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"beautiful that\" to \"beautiful, that\", inserting a comma after \"beautiful\". The core punctuation change is identical, so the text content matches the ground truth intention. Line numbers are ignored as instructed."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 33,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\nText Content Accuracy: Both edits change the same word from \"they\" to \"They\", capturing the identical core capitalization correction. Line number difference is ignored as instructed."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:36:58.135773",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:32:40.065324",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" in the phrase \"hers matters\" \u2192 \"hers, matters.\" The prediction captures the same core correction by changing \"hers\" \u2192 \"hers,\" (even though it omits the following word \"matters\"). This still reflects the essential intended edit: adding a comma immediately after \"hers.\""
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"moment\" (\"moment with\" \u2192 \"moment, with\"). The predicted edit (\"moment\" \u2192 \"moment,\") captures this same core punctuation insertion. Although it omits the following word \"with\" from the context, it still represents the essential intended correction: adding a comma after \"moment.\""
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful",
"corrected_text": "beautiful,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"beautiful\" in the phrase \"beautiful that\" \u2192 \"beautiful, that.\" The predicted edit \"beautiful\" \u2192 \"beautiful,\" captures the same essential punctuation change (adding a comma immediately after \"beautiful\"). Although it includes less surrounding context (it does not show the following word \"that\"), it still reflects the core intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core transformation: \"they\" \u2192 \"They\". The essential capitalization change is identical, with no discrepancies in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:37:03.069234",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:33:01.632494",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"hers\" in the phrase \"hers and\" \u2192 \"hers, and.\" The predicted edit \"hers\" \u2192 \"hers,\" captures the same core punctuation change (adding a comma immediately after \"hers\"). Although it omits the following word \"and\" as context, it still correctly represents the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"hers matters\" to \"hers, matters,\" which is fundamentally an insertion of a comma after \"hers.\" The predicted edit changes \"hers\" to \"hers,\", which captures the same core punctuation change (adding a comma immediately after \"hers\"). Although the prediction includes less surrounding context (it omits \"matters\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"moment\" in the phrase \"moment with\" \u2192 \"moment, with\". The predicted edit \"moment\" \u2192 \"moment,\" captures the same essential punctuation change (adding a comma immediately after \"moment\"). Although it provides less surrounding context (it omits \"with\"), it still represents the core intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful",
"corrected_text": "beautiful,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"beautiful\" in the phrase \"beautiful that\" \u2192 \"beautiful, that.\" The predicted edit \"beautiful\" \u2192 \"beautiful,\" captures the same core punctuation insertion (adding a comma immediately after \"beautiful\"). Although it provides less surrounding context (it omits \"that\"), it still represents the essential intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same core change from \"they\" to \"They\". There is no discrepancy in the original or corrected text. Therefore, the prediction captures the intended edit."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:37:04.120260",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:33:12.480687",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same correction (inserting a comma to make \"shore, and\"). The core change\u2014adding the comma after \"shore\"\u2014is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\nText Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\". This captures the identical core change with no discrepancy. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:37:13.363218",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:33:31.780525",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have the type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Thank you!\" to \"Thank you,\", replacing the exclamation mark with a comma. The core punctuation change is identical. Therefore, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after \"shore\" in the phrase \"shore and\" \u2192 \"shore, and\". The predicted edit captures the same core punctuation change by changing \"shore\" \u2192 \"shore,\". Although it omits the following word \"and\" from the context, it still correctly represents the essential intended correction: adding a comma after \"shore\"."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\". The core intended change is captured identically with no discrepancy."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "that,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy",
"corrected_text": "fancy,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:45:24.124085",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:33:56.406282",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, replacing the exclamation mark with a comma (\"Thank you!\" \u2192 \"Thank you,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after the word \"shore\" in the phrase \"shore and\" \u2192 \"shore, and\". The predicted edit captures the same core change by converting \"shore\" \u2192 \"shore,\". Although it omits the following word \"and\" from the original/corrected spans, it still represents the essential intended punctuation insertion at the correct location (immediately after \"shore\"). Therefore, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both identify the same replacement of the word \"that\" with \"than\". The core intended change is identical in the prediction and ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "that,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy",
"corrected_text": "fancy,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:45:37.657343",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:34:48.536334",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: adding a period to convert \"Chapter XXIX\" into \"Chapter XXIX.\". The original_text and corrected_text are identical between ground truth and prediction, capturing the core intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the edit type matches exactly.\nText Content Accuracy: Both specify the same core change: \"city\" is corrected to \"City\". There is no discrepancy in the original or corrected text, and the essential capitalization intent is fully captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both identify the same substitution, changing \"of\" to \"to\". The core intended change is captured without discrepancy."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same substitution, changing \"them,\" to \"them;\". This captures the core intended punctuation change with identical original and corrected text."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\nText Content Accuracy: Both edits change \"motionless\" to \"motionless,\" by adding a comma immediately after the word. The core intended punctuation change is captured exactly."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"view\" (\"view\" \u2192 \"view,\"). The original and corrected texts are identical between ground truth and prediction, so the intended punctuation insertion is captured correctly."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "lord",
"corrected_text": "lord,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:45:46.463621",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:35:31.579695",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both identify the same change from \"city\" to \"City\". The core intended correction (capitalizing the word) is captured precisely with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are of type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"of\" with \"to\". The core change is identical, with no discrepancy in original or corrected text. Therefore, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit adds a comma in the same location, shown with additional surrounding context (\"existence at\" \u2192 \"existence, at\"). This captures the same core punctuation insertion intended by the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth edit inserts a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit applies the same core change, inserting a comma after \"motionless\" in a slightly longer span of text (\"motionless when\" \u2192 \"motionless, when\"). This added context does not alter the essential correction and accurately captures the intended punctuation change."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit (\"view by\" \u2192 \"view, by\") also inserts a comma immediately after \"view.\" Although it includes extra surrounding context (the word \"by\"), it captures the same core punctuation change intended by the ground truth."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them images",
"corrected_text": "them; images",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:45:53.581943",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:35:31.253945",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\". The original_text and corrected_text are identical between ground truth and prediction, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. Text Content Accuracy: Both specify changing \"city\" to \"City\", which captures the identical core correction (capitalizing the first letter). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing the original text \"of\" with the corrected text \"to\". This captures the core intended change with identical original and corrected tokens. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same core change, replacing \"them,\" with \"them;\". The original and corrected text match the ground truth\u2019s intention with no discrepancies."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2. Text Content Accuracy: The ground truth intends to insert a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change within a slightly larger context (\"existence at\" \u2192 \"existence, at\"), which still clearly captures the essential insertion of the comma immediately after \"existence.\" Therefore, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit applies the same core change in a slightly larger context (\"motionless when\" \u2192 \"motionless, when\"), which still clearly represents inserting a comma immediately after \"motionless.\" Therefore, it captures the essential intended correction."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the predicted edit are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change within a longer span (\"view by\" \u2192 \"view, by\"), which correctly captures the essential insertion of the comma immediately after \"view\". The extra context word \"by\" does not alter the intended correction."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:45:55.387445",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:35:47.298736",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"italicize\", which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth indicates that the word \"him\" should be italicized without changing the literal characters of the word (original_text and corrected_text are both \"him\"). The prediction represents the same intention by outputting \"him\", which is a common explicit markup for italicization. Although the corrected_text differs in raw characters due to the added HTML tags, the core intended change\u2014italicizing the word \"him\"\u2014is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same core change, replacing \"authorize\" with \"authorise\". There is no discrepancy in the original or corrected text. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:46:09.381915",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:36:17.970519",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both identify the same target text \"him\" with no change in spelling (only formatting via italics). The prediction captures the same core formatting change intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both identify the same original text (\"authorize\") and the same corrected text (\"authorise\"), capturing the identical core change (American spelling to British spelling). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:46:18.978669",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:36:08.988651",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both the ground truth and prediction apply italics to the same token, \"him\" (original_text and corrected_text are identical in both). The essential change\u2014formatting \"him\" in italics\u2014is captured correctly with no mismatch in the target text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"now I\" to \"now, I\", inserting a comma after \"now\". The core punctuation correction is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"authorize\" with \"authorise\". The core intended change (American to British spelling) is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "little",
"corrected_text": "little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:46:21.123693",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:44:29.284174",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly,",
"corrected_text": "quickly",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me",
"corrected_text": "me,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:46:19.830340",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:37:57.520581",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "morning;",
"corrected_text": "morning,",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly,",
"corrected_text": "quickly",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.4",
"date": "2026-03-05T19:46:34.815396",
"annotator": "GPT-5.4",
"annotation_date": "2026-03-05T19:38:48.048188",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "a skirt",
"corrected_text": "as a skirt",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly,",
"corrected_text": "quickly",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2026-03-05",
"release_source": "https://openai.com/index/introducing-gpt-5-4/"
},
{
"model_name": "GPT-5.5",
"model_id": "gpt-5.5",
"date": "2026-04-26T00:12:49.943858",
"shots": 2,
"config": {
"model_id": "gpt-5.5",
"display_name": "GPT-5.5",
"shots": 2,
"temperature": 0.0,
"reasoning_effort": "high",
"date": "2026-04-25",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:14:37.898992",
"file_results": [
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:38.358379",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:14:37.898992",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a period to change \"Fellow Travellers\" to \"Fellow Travellers.\" The original_text and corrected_text match the ground truth intention precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the identical core change: inserting a comma after \"yonder\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change, converting \"Sir\" to \"sir\". There is no discrepancy in original or corrected text. Therefore the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"pockets and\") and the same correction (\"pockets, and\"), capturing the identical core change of inserting a comma after \"pockets\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sir, is",
"corrected_text": "Sir; is",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:29.502896",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:32:09.177225",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The core intended punctuation change is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original span (\"howling over\") and the same correction (inserting a comma to make \"howling, over\"). The core intended punctuation change is captured precisely."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), inserting a comma after \"yonder\". The core change is identical. Therefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"Sir\" to \"sir,\" capturing the identical core correction (lowercasing the initial letter). No discrepancies in original or corrected text content."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same correction (\"pockets, and\"), inserting a comma after \"pockets\". The core intended change is captured identically. Therefore, the predicted edit matches the ground truth."
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:24.653611",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:48:44.072093",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore, the prediction fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"yonder to-day\" to \"yonder, to-day\", inserting a comma after \"yonder\". The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same change from \"pockets and\" to \"pockets, and\", correctly capturing the intended insertion of a comma after \"pockets\". No discrepancy in the core edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "TRAVELLERS",
"corrected_text": "TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:23.550607",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:14:47.139291",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"No sir\") and the same corrected text (\"No, sir\"), capturing the identical core change\u2014adding a comma after \"No\". Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to capitalize the initial word in the phrase, changing \"why did he dine\" to \"Why did he dine\". The predicted edit captures the same core change by changing \"why\" to \"Why\". Although it includes less surrounding context than the ground truth, it correctly identifies and applies the essential capitalization correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the ending punctuation from an exclamation mark to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction applies the same core punctuation change on a slightly shorter span: \"stones!\" \u2192 \"stones?\". Although it omits the preceding word \"cold\", it still captures the essential intended correction (changing ! to ? on the same phrase ending with \"stones\"). Therefore, the text content matches the ground truth\u2019s intention."
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:37.910209",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:32:09.383772",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both edits change \"No sir\" to \"No, sir\" by inserting a comma after \"No.\" This captures the exact core correction intended in the ground truth with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates capitalizing the initial word of the phrase (\"why did he dine\" \u2192 \"Why did he dine\"). The predicted edit captures the core change by capitalizing just the word \"why\" \u2192 \"Why\". Although it includes less surrounding context than the ground truth, it correctly represents the essential intended correction (capitalizing the initial 'w' in 'why')."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the ending punctuation from an exclamation point to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction captures the same core change on the word \"stones!\" \u2192 \"stones?\". Although it omits the preceding word \"cold,\" it still reflects the essential intended edit: replacing \"!\" with \"?\" at the end of the phrase. Therefore, the text content matches the ground truth's intention."
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:38.489638",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:47:48.315486",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"No sir\") and the same corrected text (\"No, sir\"), capturing the identical core change\u2014adding a comma after \"No\". Therefore the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word of the phrase (\"why\" \u2192 \"Why\") within the larger context \"why did he dine\" \u2192 \"Why did he dine\". The prediction captures this core change by specifying \"why\" \u2192 \"Why\". Although it includes less surrounding context, it correctly represents the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the ending punctuation from an exclamation mark to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction captures the same core change\u2014replacing \"!\" with \"?\"\u2014but applies it to the shorter span \"stones!\" \u2192 \"stones?\". This still reflects the essential intended correction (changing the sentence-ending punctuation after \"stones\"), and omitting the preceding word \"cold\" is acceptable as missing context, not a different edit.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:26.302630",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:14:16.408445",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\n2) Text Content Accuracy: The ground truth changes \"if he\" to \"If he,\" which is specifically capitalizing the word \"if\" at the start of the phrase. The prediction changes \"if\" to \"If\"\u2014it captures the same core change (lowercase to uppercase for \"if\"). Although it omits the following word \"he\" as context, the essential intended correction is still accurately represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\" edits, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the verb phrase \"said Clennam\" to \"asked Clennam\". The prediction replaces only \"said\" with \"asked\", leaving \"Clennam\" implicit/unchanged. This captures the core intended change (said \u2192 asked). Additional surrounding context is optional, so omitting \"Clennam\" does not alter the essential correction."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:25.457356",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:32:01.581165",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth.\n\n2) Text Content Accuracy: The ground truth change is capitalizing the initial word in the phrase: \"if he\" \u2192 \"If he\". The predicted edit \"if\" \u2192 \"If\" captures the same core correction (capitalizing \"if\"). Although it includes less surrounding context (omits \"he\"), it still reflects the essential intended change.\n\nTherefore, both criteria are satisfied."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the core intended change by replacing \"said\" \u2192 \"asked\" (even though it omits the surrounding context \"Clennam\"). Since the essential word substitution is correctly identified and the remaining context is unchanged, the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:56.335529",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:47:26.735257",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 37,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the word \"if\" at the start of the phrase \"if he\" (\"if he\" \u2192 \"If he\"). The prediction captures the same core change by converting \"if\" to \"If.\" Although it omits the following word \"he\" from the original/corrected_text fields, it still represents the essential intended correction (capitalizing \"if\")."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 40,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction are a \"replacement\" edit, so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes the verb in the phrase \"said Clennam\" to \"asked Clennam\"\u2014the core intended correction is \"said\" \u2192 \"asked\". The prediction captures this same essential change, even though it includes less surrounding context (it replaces only \"said\" with \"asked\"). Omitting \"Clennam\" does not change the core correction being made.\n\nTherefore, the predicted edit matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:09:28.145655",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:15:09.426173",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both identify the same original text (\"high road\") and the same correction (\"high-road\"), capturing the identical core change of inserting a hyphen between the words. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"materials\" to change \"materials costing\" to \"materials, costing.\" The original and corrected texts are identical between ground truth and prediction, so the intended edit is fully captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same correction (\"about, and\"), capturing the identical core change of inserting a comma after \"about\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change: inserting a comma after \"earnestly\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same original segment (\"said when\") and the same correction (\"said, when\"), capturing the identical core punctuation insertion (adding a comma after \"said\"). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the identical core change: \"said\" \u2192 \"said,\", i.e., inserting a comma after the word \"said\". There is no discrepancy in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original segment (\"Clennam in\") and the same correction (\"Clennam, in\"), inserting a comma after \"Clennam\". The core intended punctuation change is captured exactly."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:31.578911",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:32:25.551149",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both identify the same original text (\"high road\") and the same correction (\"high-road\"), capturing the intended hyphenation change precisely. No discrepancies in the core edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original segment \"materials costing\" and the same correction \"materials, costing,\" inserting a comma after \"materials.\" This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same correction (\"about, and\"), capturing the identical core change: inserting a comma after \"about.\" Therefore, the prediction matches the ground truth's intended edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change of inserting a comma after \"earnestly\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the identical transformation from \"said when\" to \"said, when,\" inserting a comma after \"said.\" The core punctuation change is captured exactly with no discrepancy."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the edit type matches exactly.\n2. Text Content Accuracy: Both specify the same change from \"said\" to \"said,\", correctly capturing the intended punctuation insertion (a comma after \"said\"). No discrepancies in the core edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"Clennam in\" to \"Clennam, in\" by inserting a comma after \"Clennam.\" This captures the identical core correction with matching original and corrected text.\n\nTherefore, the prediction correctly reflects the ground truth edit."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:32.531910",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:48:16.003258",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"high road\" is corrected to \"high-road\" by inserting a hyphen. The original_text and corrected_text are identical between ground truth and prediction, capturing the intended punctuation change precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change (inserting a comma after \"materials\"). Therefore the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma so that \"about and\" becomes \"about, and.\" The original and corrected text match exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the edit type matches exactly. Text Content Accuracy: Both specify the same transformation from \"earnestly telling\" to \"earnestly, telling\", inserting a comma after \"earnestly\". The core punctuation change is identical. Therefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"said when\") and the same corrected text (\"said, when\"), capturing the identical core change: insertion of a comma after \"said\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core change, adding a comma after \"said\" (\"said\" \u2192 \"said,\"). There is no discrepancy in the original or corrected text. Therefore, the prediction fully captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same corrected text (\"Clennam, in\"), capturing the identical core change\u2014adding a comma after \"Clennam\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:25.284004",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:14:00.696096",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a final period. This captures the exact core correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:21.034297",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:31:48.845013",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a final period. The core change is identical, with no discrepancies in original or corrected text. Therefore, the prediction fully captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:25.443137",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:47:13.039125",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both apply the same core change\u2014adding a period at the end of \"Sun and Shadow\" to make it \"Sun and Shadow.\". The original and corrected strings match the ground truth exactly. Therefore, the predicted edit correctly captures the intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:30.853101",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:14:52.771615",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The original and corrected texts are identical between ground truth and prediction, capturing the core intended change precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "One man slowly",
"corrected_text": "One man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth change inserts a comma after \"man\" in the phrase \"man slowly\" \u2192 \"man, slowly\". The prediction inserts the same comma in a larger context: \"One man slowly\" \u2192 \"One man, slowly\". This preserves the core intended punctuation edit (comma after \"man\") while adding extra surrounding context, which is allowed. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the types match exactly.\n\nText Content Accuracy: Both ground truth and prediction apply the same core change: inserting a comma after \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). The original and corrected texts are identical between the two, so the intended punctuation edit is captured correctly."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:38.882246",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:33:07.373784",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit specify the type as \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The original_text and corrected_text are identical between ground truth and prediction, so the core change is captured precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both change the original text \"Saone\" to the corrected text \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core intended correction is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"man slowly\" to \"man, slowly,\" inserting a comma after \"man.\" The core change is identical, with no deviation in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"Chalons\" (\"Chalons was\" \u001a \"Chalons, was\"). The original and corrected text strings are identical between ground truth and prediction, so the intended edit is fully captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:49.193452",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:48:02.519895",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Saone\") and the same corrected text (\"Sa\u00f4ne\"), capturing the exact intended change (adding the circumflex over the 'o'). Therefore, the core edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core textual change is identical. Therefore, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both edits change \"man slowly\" to \"man, slowly,\" inserting a comma after \"man.\" The core punctuation change is identical, with no discrepancies in original or corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same correction (\"Chalons, was\"), capturing the intended insertion of a comma after \"Chalons.\" The core change is identical."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:57.278068",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:16:26.398624",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2. Text Content Accuracy: Both specify the same transformation from \"heart so\" to \"heart, so\" (inserting a comma after \"heart\"). The core change is identical, with no discrepancies in original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling;",
"corrected_text": "smiling,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "did",
"corrected_text": "did indeed",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:10:59.377511",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:33:47.131380",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the identical change from \"heart so\" to \"heart, so\" (inserting a comma after \"heart\"). The core punctuation correction is captured exactly with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "heart (as",
"corrected_text": "heart indeed (as",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling:",
"corrected_text": "smiling,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:11:03.595169",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:49:16.070835",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"heart so\") and the same corrected text (\"heart, so\"), capturing the identical core change of inserting a comma after \"heart.\" Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "(as she",
"corrected_text": "(as indeed she",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling;",
"corrected_text": "smiling,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "That's well! I hoped so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:11:22.919413",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:14:46.792578",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change (inserting a comma after \"hers\"). Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same transformation from \"hers matters\" to \"hers, matters,\" inserting a comma after \"hers.\" The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the identical transformation from \"moment with\" to \"moment, with\", inserting a comma after \"moment\". The core punctuation change is captured exactly with no discrepancy."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"beautiful that\" to \"beautiful, that\", inserting a comma after \"beautiful\". The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: converting \"they\" to \"They\". There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:11:23.050693",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:32:00.609751",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both identify the same original text (\"hers and\") and the same correction (\"hers, and\"), inserting a comma after \"hers.\" The core change is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction make the same core change, inserting a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The original and corrected text strings match exactly, so the intended edit is correctly captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original segment \"moment with\" and the same correction \"moment, with,\" inserting a comma after \"moment.\" The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"beautiful that\") and the same corrected text (\"beautiful, that\"), inserting a comma after \"beautiful\". This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: \"they\" is corrected to \"They\". There is no discrepancy in original or corrected text. Therefore, the predicted edit fully matches the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:11:30.058856",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:47:37.701164",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"hers and\" to \"hers, and\" (inserting a comma after \"hers\"). The core punctuation correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers.\" The core punctuation correction is identical."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both apply the same core change\u2014adding a comma after \"moment\" in the phrase \"moment with,\" converting \"moment with\" to \"moment, with.\" The original and corrected text strings are identical between ground truth and prediction, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: inserting a comma between \"beautiful\" and \"that\" (\"beautiful that\" \u2192 \"beautiful, that\"). The core change is identical.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change from \"they\" to \"They\". There is no discrepancy in the original or corrected text, so the intended capitalization correction is accurately captured."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:11:44.513727",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:15:34.444026",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have the type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original_text (\"shore and\") and the same corrected_text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth specifies a \"replacement\" edit, and the prediction also uses \"replacement\", so the edit type matches exactly. Text Content Accuracy: Both ground truth and prediction replace the word \"that\" with \"than\". The core change is identical with no discrepancy in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:11:45.622972",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:33:16.451149",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"shore and\") and the same correction (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth specifies a \"replacement\" edit, and the predicted edit is also a \"replacement\", so the type matches exactly.\n\nText Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\". This captures the identical core change with no discrepancy. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:11:49.125881",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:48:39.553799",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same original_text (\"shore and\") and the same corrected_text (\"shore, and\"), capturing the identical core change of inserting a comma after \"shore\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both specify replacing the original text \"that\" with the corrected text \"than\". The core change is identical with no divergence in the words being replaced or the replacement word. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:12:18.074760",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:16:31.615976",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"capitalization,\" which matches the ground truth exactly.\nText Content Accuracy: The predicted change is \"city\" \u2192 \"City,\" which is identical to the ground truth and captures the core intended capitalization correction. No discrepancies or missing/extra changes are present."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the types match exactly.\n\nText Content Accuracy: Both ground truth and prediction replace the original text \"of\" with the corrected text \"to\". This captures the same core change with no discrepancy in the substituted words.\n\nTherefore, the predicted edit correctly matches the ground truth in both type and content."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change within a slightly larger span of text (\"existence at\" \u2192 \"existence, at\"), which still places the comma immediately after \"existence\". This captures the essential intended punctuation edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction adds the same comma in context (\"motionless when\" \u2192 \"motionless, when\"), which captures the identical core punctuation change. Including the following word \"when\" as additional context does not alter the essential edit.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction uses a larger span (\"view by\" \u2192 \"view, by\"), but the core modification is the same: a comma is inserted immediately after \"view\". The additional context word \"by\" does not change the essential edit. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming,",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:12:45.952651",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:33:34.161147",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction change \"Chapter XXIX\" to \"Chapter XXIX.\" by adding a period at the end. The core punctuation change is identical, with matching original and corrected text.\n\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same core change from \"city\" to \"City\" with identical original and corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing the original text \"of\" with the corrected text \"to\". The core change is identical and fully captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change in a slightly wider context (\"existence at\" \u2192 \"existence, at\"), which still inserts a comma immediately after \"existence.\" This captures the essential intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches.\n\n2) Text Content Accuracy: The ground truth change is specifically changing the punctuation after \"it\" from a comma to a semicolon (\"it,\" \u2192 \"it;\"). The prediction expands the span to include the following word (\"it to\" \u2192 \"it; to\"), but the essential effect is the same: the punctuation immediately after \"it\" is set to a semicolon. Although the prediction does not explicitly show the original comma, it still captures the core intended correction (semicolon after \"it\")."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction adds that same comma after \"motionless\" but includes additional context (\"motionless when\" \u2192 \"motionless, when\"). This captures the same core punctuation correction intended by the ground truth.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth\u2019s core intent is to change the punctuation immediately following \u201cstand-still\u201d from a comma to a semicolon (\"stand-still,\" \u2192 \"stand-still;\"). The predicted edit also results in a semicolon directly after \u201cstand-still\u201d (\"stand-still; to\"). Although the predicted \"original_text\" omits the comma and includes the following word \u201cto\u201d (extra context), the essential intended change\u2014having \u201cstand-still\u201d followed by a semicolon instead of a comma\u2014is correctly captured in the corrected text."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"view\" (\"view\" \u2192 \"view,\"). The prediction applies the same core change within a longer span (\"view by\" \u2192 \"view, by\"), which still places the comma right after \"view.\" The additional context word \"by\" does not alter the essential correction.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING,",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:12:49.943858",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:49:25.645447",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XXIX",
"corrected_text": "XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period at the end of the chapter heading: \"Chapter XXIX\" \u2192 \"Chapter XXIX.\". The prediction captures the same core punctuation change by adding a period after the Roman numeral: \"XXIX\" \u2192 \"XXIX.\". Although it omits the preceding word \"Chapter\" as context, it still accurately represents the essential intended correction (adding the terminal period to the heading)."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"DREAMING\" to \"DREAMING.\", correctly capturing the intended addition of a period. Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify changing \"city\" to \"City\", which captures the same core capitalization change with identical original and corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly.\n\nText Content Accuracy: The ground truth replaces \"of\" with \"to\". The prediction also replaces \"of\" with \"to\". The core intended change is identical, with no discrepancy in the original or corrected text.\n\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2. Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit expands the span to include the following word (\"existence at\" \u2192 \"existence, at\"), but it still performs the same core change: inserting a comma immediately after \"existence.\" Additional surrounding context is allowed, and the essential punctuation correction is correctly captured."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is replacing the punctuation immediately following \"it\" from a comma to a semicolon (\"it,\" \u2192 \"it;\"). The prediction applies the same core punctuation change by making the text after \"it\" use a semicolon (\"it to\" \u2192 \"it; to\"). Although the prediction includes extra context (the following word \"to\") and does not explicitly show the comma in its original_text span, it still captures the essential intended correction: a semicolon should follow \"it\"."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change, adding a comma immediately after \"motionless\" within a longer span (\"motionless when\" \u2192 \"motionless, when\"). The additional context word \"when\" does not alter the essential intended correction; it still captures the comma insertion after \"motionless\"."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit applies the same essential change, inserting a comma immediately after \"view\" within a larger span (\"view by\" \u2192 \"view, by\"). This added context does not alter the core intention; it still captures the comma insertion after \"view.\""
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:12:07.889782",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:16:34.506840",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"italicize,\" which matches the ground truth exactly.\n\n2. Text Content Accuracy: The ground truth indicates that the word \"him\" should be italicized without changing the literal characters (original_text and corrected_text both \"him\"). The prediction represents this same intention by rendering the corrected text as \"him\", which is a conventional explicit encoding of italics. The core change\u2014apply italics to \"him\"\u2014is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n\nText Content Accuracy: Both specify replacing \"authorize\" with \"authorise\". The core intended change (American to British spelling) is captured identically, with no discrepancy in original or corrected text.\n\nTherefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to",
"corrected_text": "now, to",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:12:10.050937",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:33:20.259934",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\nText Content Accuracy: Both specify the same original_text (\"authorize\") and the same corrected_text (\"authorise\"), capturing the identical core change from American to British spelling. Therefore, the text content matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to",
"corrected_text": "now, to",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:12:15.861276",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:49:43.030038",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"italicize,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth indicates that the word \"him\" should be italicized; because italicization is a formatting change, the plain-text \"corrected_text\" may remain \"him\". The prediction represents the same intended formatting change by wrapping the corrected text as \"him\" while keeping the target word identical. This captures the core intention (italicize the word \"him\")."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\nText Content Accuracy: Both replace the same original word \"authorize\" with the same corrected word \"authorise\". The core change (American to British spelling) is captured identically.\nTherefore, the predicted edit matches the ground truth in both type and content."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to",
"corrected_text": "now, to",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:12:02.457201",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:15:52.092334",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The predicted edit likewise inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it stops at \"Mr.\" rather than \"Mr. Pancks\"), it captures the core intended punctuation change in the correct location."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". The core change is identical, with no discrepancy in original or corrected text. Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:12:14.038929",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:34:26.099848",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"For Mr. Pancks\" to \"For, Mr. Pancks\", i.e., it inserts a comma after \"For\". The prediction changes \"For Mr.\" to \"For, Mr.\", which also inserts a comma after \"For\". Although the prediction includes less surrounding context (it omits \"Pancks\"), it still captures the essential intended punctuation change in the correct location. Therefore, the core edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that,\" inserting a comma after \"quickly.\" This captures the identical core punctuation change with no discrepancies."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.5",
"date": "2026-04-26T00:12:15.131867",
"annotator": "GPT-5.5",
"annotation_date": "2026-04-25T23:49:48.407470",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks.\" The prediction likewise changes \"For Mr.\" \u2192 \"For, Mr.,\" which captures the same core punctuation change (inserting a comma immediately after \"For\" before \"Mr.\"). Although the prediction includes less surrounding context (it stops at \"Mr.\" rather than \"Mr. Pancks\"), it correctly reflects the essential intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"quickly that\") and the same corrected text (\"quickly, that\"), capturing the identical core change: insertion of a comma after \"quickly\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2026-04-23",
"release_source": "https://openai.com/index/introducing-gpt-5-5/"
},
{
"model_name": "GPT-5.5 Pro (OR)",
"model_id": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:32.453675",
"shots": 2,
"config": {
"model_id": "or_openai_gpt_5_5_pro",
"display_name": "GPT-5.5 Pro (OR)",
"shots": 2,
"temperature": 0.0,
"date": "2026-04-26",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:17:49.242594",
"file_results": [
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:43.612597",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:17:49.242594",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both ground truth and prediction change \"howling over\" to \"howling, over\" by inserting a comma after \"howling\". The essential punctuation change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both ground truth and prediction change \"yonder to-day\" to \"yonder, to-day\", inserting a comma after \"yonder\". The original and corrected texts are identical between the two, capturing the core intended punctuation change precisely."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both ground truth and prediction change \"Sir\" to \"sir\". This captures the identical core correction (lowercasing the initial letter). No discrepancies in original or corrected text are present.\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: Both ground truth and prediction change \"pockets and\" to \"pockets, and\" by inserting a comma after \"pockets.\" This captures the exact intended core punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "TRAVELLERS",
"corrected_text": "TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:45.218996",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:36:42.861974",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a period to the end of \"Fellow Travellers\" to make it \"Fellow Travellers.\" The original_text and corrected_text align perfectly with the ground truth intention. Therefore, the predicted edit is correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling.\" Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original_text (\"yonder to-day\") and the same corrected_text (\"yonder, to-day\"), capturing the identical core change of inserting a comma after \"yonder\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"Sir\" to \"sir\", capturing the identical core lowercase correction. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"pockets and\" to \"pockets, and\"\u2014inserting a comma after \"pockets.\" This captures the core intended change with no discrepancy."
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:43.558684",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:52:28.392489",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change the phrase from \"yonder to-day\" to \"yonder, to-day\", inserting a comma after \"yonder\". The original and corrected texts are identical between ground truth and prediction, so the core change is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"Sir\" to \"sir\". The core intended change (lowercasing the initial letter) is captured identically.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"pockets and\") and the same corrected text (\"pockets, and\"), capturing the identical core change\u2014adding a comma after \"pockets.\" Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:44.664541",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:19:15.472898",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"No sir\" to \"No, sir\" (inserting a comma after \"No\"). The core correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial \"w\" in the phrase \"why did he dine\" \u2192 \"Why did he dine.\" The prediction applies the same core change by capitalizing \"why\" \u2192 \"Why.\" Although it includes less surrounding context, it captures the essential intended correction (initial word capitalization)."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the ending punctuation from an exclamation mark to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction applies the same core punctuation change (! \u2192 ?) but on the slightly shorter substring \"stones!\" \u2192 \"stones?\". This still captures the essential intended correction (replacing the exclamation point with a question mark at the end of the expression), and omitting the preceding word \"cold\" is acceptable since additional surrounding context is not required as long as the core change is preserved."
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:43.389212",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:36:33.493923",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"No sir\" to \"No, sir\" by inserting a comma after \"No\". The core punctuation correction is identical, with no mismatch in original or corrected text. Therefore the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the initial \"why\" in the phrase \"why did he dine\" to \"Why\" (capitalizing the first word). The prediction captures this same core change by editing \"why\" \u2192 \"Why\". Although it includes less surrounding context than the ground truth, it correctly represents the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation mark to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction applies the same core change on a slightly narrower span: \"stones!\" \u2192 \"stones?\". This still captures the essential intended correction (changing ! to ? after \"stones\"). The omitted word \"cold\" is just additional context and does not alter the core punctuation edit."
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:44.027294",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:51:16.832681",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"No sir\") and the same correction (\"No, sir\"), adding a comma after \"No\". The predicted edit captures the identical core punctuation change intended by the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word of the phrase, changing \"why did he dine\" to \"Why did he dine.\" The prediction captures the same core change by capitalizing \"why\" to \"Why.\" Although it provides less surrounding context than the ground truth, it correctly reflects the essential intended correction (capitalizing the first word)."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation point to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction applies the same core change to a slightly narrower span: \"stones!\" \u2192 \"stones?\". This still captures the essential intended edit (replacing ! with ? at the end of the same word/phrase). The omitted preceding word \"cold\" is additional context, not part of the core punctuation change.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:45.164560",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:15:10.074375",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to capitalize the initial \"if\" in the phrase \"if he\" \u2192 \"If he.\" The prediction captures the same essential change by converting \"if\" \u2192 \"If.\" Although it includes less surrounding context (omits \"he\"), it still reflects the core intended correction: capitalizing the word \"if\" at the start of the phrase/sentence."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"said Clennam\" to \"asked Clennam\", whose core substantive change is replacing the word \"said\" with \"asked\" while leaving \"Clennam\" unchanged. The prediction captures this essential substitution by replacing \"said\" with \"asked\". Although it includes less surrounding context (omitting \"Clennam\"), it still reflects the same intended correction."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:41.581708",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:32:59.718118",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the word \"if\" at the start of the phrase (\"if he\" \u2192 \"If he\"). The prediction captures the same core change by capitalizing \"if\" (\"if\" \u2192 \"If\"). Although it includes less surrounding context (omits \"he\"), the essential intended correction\u2014capitalizing \"if\"\u2014is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes the verb \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the same core substitution by replacing \"said\" with \"asked\" (with less surrounding context). Since the essential intended change is the verb replacement and the remaining context (\"Clennam\") is unchanged, the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:44.323068",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:49:17.393374",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"if he\" to \"If he,\" which capitalizes the word \"if\" at the beginning of the phrase. The predicted edit changes \"if\" to \"If,\" capturing the same core correction (capitalizing \"if\"). The omission of the following word \"he\" is acceptable because it is additional context and does not alter the essential intended change.\n\nTherefore, the predicted edit correctly captures the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes \"said Clennam\" to \"asked Clennam\", i.e., the substantive correction is replacing the word \"said\" with \"asked\" while keeping \"Clennam\" unchanged. The predicted edit replaces \"said\" with \"asked\" without including the surrounding \"Clennam\" context. This still captures the core intended change (\"said\" \u2192 \"asked\"), and omission of extra unchanged context is acceptable under the criteria."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:23:52.055234",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:20:31.156082",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core change from \"high road\" to \"high-road\" (inserting a hyphen between the words). There is no discrepancy in original or corrected text. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), inserting a comma after \"materials.\" This captures the core intended punctuation change with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same correction (\"about, and\"), capturing the identical core change\u2014adding a comma after \"about\". Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original phrase \"earnestly telling\" and the same correction \"earnestly, telling\", inserting a comma after \"earnestly\". The core change is identical.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change: inserting a comma after \"said\" in the phrase \"said when\" to become \"said, when\". The original and corrected texts are identical between ground truth and prediction, so the core intended change is fully captured."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: \"said\" \u2192 \"said,\" (adding a comma after \"said\"). There is no discrepancy in the original or corrected text, so the prediction fully captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same correction (\"Clennam, in\"), capturing the identical core change\u2014adding a comma after \"Clennam.\" Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:17.430637",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:36:03.042210",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change from \"high road\" to \"high-road\" (inserting a hyphen). The original and corrected texts are identical between ground truth and prediction, so the intended punctuation change is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change of inserting a comma after \"materials.\" Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same corrected text (\"about, and\"), capturing the identical core change\u2014adding a comma after \"about\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change\u2014adding a comma after \"earnestly.\" Therefore, the prediction fully matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same transformation from \"said when\" to \"said, when,\" inserting a comma after \"said.\" The core punctuation change is identical. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: \"said\" \u2192 \"said,\" (adding a comma after \"said\"). There is no discrepancy in original or corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same correction (\"Clennam, in\"), capturing the identical core change\u2014adding a comma after \"Clennam.\" Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:17.182500",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:52:02.784640",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road\" (inserting a hyphen). The core change is identical with no discrepancy. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change\u2014adding a comma after \"materials.\" Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same correction (\"about, and\"), capturing the identical core change\u2014adding a comma after \"about\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change of inserting a comma after \"earnestly\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly. 2) Text Content Accuracy: Both specify the identical transformation from \"said when\" to \"said, when,\" inserting a comma after \"said.\" This captures the core intended punctuation change with no discrepancy."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: \"said\" is corrected to \"said,\" by adding a comma. There is no discrepancy in original or corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change, inserting a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The original and corrected text strings match the ground truth exactly, capturing the intended edit."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:06.621646",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:15:36.277423",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. The core punctuation change is captured identically.\nTherefore, the predicted edit matches the ground truth in both type and content."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:14:29.187569",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:35:08.585645",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW",
"corrected_text": "SUN AND SHADOW.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:14:29.289343",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:48:57.883710",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW",
"corrected_text": "SUN AND SHADOW.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:18.466856",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:18:58.999119",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify replacing \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core textual change is identical, with no mismatch in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\nText Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'), which captures the same core correction. No discrepancies in the original or corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the identical transformation from \"man slowly\" to \"man, slowly,\" inserting a comma after \"man.\" This captures the core intended correction with no discrepancy."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change: insertion of a comma after \"Chalons\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:17.634777",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:37:14.592013",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"replacement\", so the type matches exactly. 2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'), capturing the identical core correction. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\nText Content Accuracy: Both change the original text \"Saone\" to the corrected text \"Sa\u00f4ne\" (adding the circumflex over the 'o'), which captures the exact core correction intended. No discrepancy in the content of the change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original and corrected text\u2014changing \"man slowly\" to \"man, slowly\" by inserting a comma after \"man.\" This captures the core intended change with no discrepancies."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). The original and corrected texts match exactly between ground truth and prediction, capturing the intended punctuation insertion."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:24.151419",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:51:38.200684",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the original text \"Saone\" with the corrected text \"Sa\u00f4ne\" (adding the circumflex over the o). The core change is identical.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n\nText Content Accuracy: Both specify the same original text \"Saone\" and the same corrected text \"Sa\u00f4ne\" (adding the circumflex over the 'o'). This captures the core intended change precisely with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same transformation: \"man slowly\" \u2192 \"man, slowly,\" inserting a comma after \"man.\" This captures the core intended punctuation change with no discrepancy."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit have the type \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change\u2014adding a comma after \"Chalons\". Therefore, the predicted edit matches the ground truth\u2019s intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:14.650208",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:23:56.108450",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"heart so\") and the same corrected text (\"heart, so\"), capturing the identical core change\u2014adding a comma after \"heart\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "as she",
"corrected_text": "as indeed she",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches.\n\n2) Text Content Accuracy: The ground truth inserts the word \"indeed\" into the phrase, changing \"as she did\" to \"as indeed she did.\" The prediction also reflects insertion of \"indeed\" after \"as\" (\"as she\" \u2192 \"as indeed she\"). Although the prediction truncates the surrounding context (omitting \"did\"), it captures the core intended change: adding \"indeed\" in that location. Therefore, the essential edit content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling;",
"corrected_text": "smiling,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:18.870672",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:40:12.853045",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so much",
"corrected_text": "heart, so much",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). The prediction also inserts a comma after \"heart\" and simply includes additional following context (\"heart so much\" \u2192 \"heart, so much\"). The essential punctuation change is the same, so the prediction captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"insertion\", so the type matches exactly.\n\nText Content Accuracy: Both identify the same original text (\"as she did\") and the same corrected text (\"as indeed she did\"), capturing the core change of inserting \"indeed\". No discrepancies in the intended modification."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling:",
"corrected_text": "smiling;",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:14.047523",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:59:52.998392",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"heart so\") and the same correction (\"heart, so\"), inserting a comma after \"heart.\" This captures the identical core change with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": 22,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same change: removing the comma in \"generous, Mr.\" to produce \"generous Mr.\". The original_text and corrected_text match the ground truth\u2019s intended punctuation correction exactly.\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "did,",
"corrected_text": "did indeed,",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling;",
"corrected_text": "smiling,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:32.919232",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:18:27.280112",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"hers and\" to \"hers, and,\" correctly capturing the intended insertion of a comma after \"hers.\" There is no discrepancy in the core change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same correction (\"hers, matters\"), inserting a comma after \"hers.\" The predicted edit captures the core intended punctuation change with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits change the same original text \"moment with\" to the same corrected text \"moment, with\", inserting a comma after \"moment\". The core intended punctuation change is captured precisely."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change: inserting a comma to revise \"beautiful that\" to \"beautiful, that\". The core intended punctuation correction is captured identically."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"they\" is corrected to \"They\". There is no difference in original or corrected text, so the essential edit is captured precisely.\nTherefore, the predicted edit matches the ground truth intent."
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:46.080976",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:34:18.893585",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"hers and\" to \"hers, and\" (inserting a comma after \"hers\"). The core punctuation correction is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"hers\" so that \"hers matters\" becomes \"hers, matters.\" The original_text and corrected_text align with the ground truth\u2019s intention with no discrepancies."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"moment with\" to \"moment, with,\" inserting a comma after \"moment.\" This captures the identical core change with no discrepancies."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same correction (\"beautiful, that\"), capturing the identical core change: insertion of a comma after \"beautiful\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\nText Content Accuracy: The predicted change is \"they\" \u2192 \"They,\" which matches the ground truth correction exactly and captures the same core capitalization change. No discrepancies in the original or corrected text."
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:50.220660",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:49:00.107834",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"hers and\" to \"hers, and,\" inserting a comma after \"hers.\" The core punctuation correction is identical."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change: insertion of a comma after \"hers.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit specify the type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"moment with\" to \"moment, with\" by inserting a comma after \"moment\". The original and corrected text strings are identical between ground truth and prediction, capturing the same core punctuation change."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: Both edits change \"beautiful that\" to \"beautiful, that\" (inserting a comma after \"beautiful\"). The core punctuation change is identical in the prediction and ground truth.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\nText Content Accuracy: Both specify the same core change from \"they\" to \"They\" (capitalizing the first letter). There is no discrepancy in original or corrected text. Therefore, the prediction captures the intended correction."
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:49.132310",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:21:36.044804",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, replacing the exclamation mark in \"Thank you!\" with a comma to form \"Thank you,\", capturing the identical intended correction. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the identical core change from \"shore and\" to \"shore, and\" (inserting a comma after \"shore\"). There is no mismatch or missing/extra substantive change. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the original text \"that\" with the corrected text \"than\". The core intended change is identical, with no discrepancy in the target or replacement text.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:57.410601",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:38:09.671980",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same correction (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both identify the same substitution of the word \"that\" with \"than\". The core intended change is identical, with no deviation in original or corrected text. Therefore, the prediction fully matches the ground truth intent."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:24:49.392472",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:54:08.693977",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"Thank you!\" to \"Thank you,\", capturing the identical core punctuation change (replacing an exclamation mark with a comma). Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"shore and\" and the same corrected text \"shore, and\", capturing the identical core change (inserting a comma after \"shore\"). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\". The core intended change is identical in the prediction and the ground truth.\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:28.703830",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:24:34.717405",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the identical transformation from \"DREAMING\" to \"DREAMING.\" (adding a period). The core intended change is captured precisely with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\nText Content Accuracy: The ground truth changes \"city\" to \"City\". The predicted edit specifies the same original text (\"city\") and the same corrected text (\"City\"), capturing the identical core capitalization change.\n\nTherefore, the prediction fully matches the ground truth intent."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing the original text \"of\" with the corrected text \"to\". The core change is identical, with no discrepancy in the replaced or replacement text.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The prediction applies the same core change, adding a comma after \"existence\" within a slightly larger context span (\"existence at\" \u2192 \"existence, at\"). This captures the essential intended punctuation correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same essential punctuation change, showing additional context (\"motionless when\" \u2192 \"motionless, when\"). This captures the core intended edit: adding a comma immediately after \"motionless.\""
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma after \"view\" as well, even though it includes additional surrounding context (\"view by\" \u2192 \"view, by\"). This captures the same core punctuation change intended by the ground truth.\n\nTherefore, the prediction correctly reflects the ground truth edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "xxix",
"corrected_text": "xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:32.453675",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-26T00:00:03.908138",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XXIX",
"corrected_text": "XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates adding a period to the chapter heading, changing \"Chapter XXIX\" to \"Chapter XXIX.\" The prediction changes \"XXIX\" to \"XXIX.\" This captures the core intended edit (inserting a period after the Roman numeral). Although the prediction omits the preceding word \"Chapter\" from the span it edits, it still reflects the same essential punctuation change at the correct location."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: adding a period to \"DREAMING\" to make it \"DREAMING.\". The original and corrected texts are identical between ground truth and prediction, so the intended correction is captured correctly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change the word \"city\" to \"City,\" capturing the same core capitalization correction with identical original and corrected text.\nTherefore, the prediction correctly matches the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing the original text \"of\" with the corrected text \"to\". The core change is identical and fully captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit applies the same core punctuation change, adding a comma after \"existence\" within a slightly larger context span (\"existence at\" \u2192 \"existence, at\"). This captures the essential intended correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core punctuation change, adding a comma immediately after \"motionless\" while including additional surrounding context (\"motionless when\" \u2192 \"motionless, when\"). This preserves the essential intended edit.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma immediately after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma after \"view\" within a longer context string (\"view by\" \u2192 \"view, by\"). This captures the same core punctuation insertion at the same location, merely including an extra following word as context. Therefore, the essential intended edit is correctly represented."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is specifically to replace the comma after \"existence\" with a semicolon (\"existence,\" \u2192 \"existence;\"). The predicted edit applies a semicolon immediately after \"existence\" (\"existence is\" \u2192 \"existence; is\"), which captures the same core intention: the punctuation following \"existence\" should be a semicolon rather than the original comma. The additional surrounding word (\"is\") is extra context and does not change the essential correction being made."
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "lord and",
"corrected_text": "lord, and",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:29.275496",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-26T00:12:50.802677",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"Chapter XXIX\" to \"Chapter XXIX.\" by adding a period at the end. The core intended punctuation change is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the identical core change: adding a period to \"DREAMING\" to form \"DREAMING.\". Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"capitalization\" and the predicted type is also \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"city\" is corrected to \"City\". There is no discrepancy in the original or corrected text, so the essential edit is captured accurately."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "return of",
"corrected_text": "return to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction are a \"replacement\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"of\" to \"to\". The prediction changes \"return of\" to \"return to\", which includes extra surrounding context but preserves the same core substitution (\"of\" \u2192 \"to\"). This correctly captures the intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit applies the same core punctuation change in a slightly larger context span (\"existence at\" \u2192 \"existence, at\"), which still places the comma immediately after \"existence.\" This captures the essential intended correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core punctuation change by inserting a comma immediately after \"motionless\" within a slightly longer context (\"motionless when\" \u2192 \"motionless, when\"). This correctly captures the essential intended edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit applies the same core change in a slightly larger context (\"view by\" \u2192 \"view, by\"), which still places the comma immediately after \"view.\" This captures the essential intended correction."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it to",
"corrected_text": "it; to",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still to",
"corrected_text": "stand-still; to",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence is",
"corrected_text": "existence; is",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:14.604872",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:25:24.544289",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"italicize,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth indicates that the word \"him\" should be italicized (original and corrected text are both \"him\" because the change is purely formatting). The prediction represents the same formatting change by encoding italics as \"him.\" Although the corrected_text differs in markup representation, it captures the same core intention: apply italics to the word \"him.\""
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"now I\" to \"now, I,\" inserting a comma after \"now.\" The core change is identical, so the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the same original text \"authorize\" with the same corrected text \"authorise\". The core change (American to British spelling) is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to",
"corrected_text": "now, to",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:17.184321",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:45:27.702985",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly. Text Content Accuracy: Both specify the same target text \"him\" to be italicized (original_text and corrected_text are identical in both, indicating a formatting change only). The prediction captures the same core change as the ground truth. Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both edits replace \"authorize\" with \"authorise\". This captures the identical core change (American to British spelling). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to",
"corrected_text": "now, to",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:21.941996",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-26T00:00:03.831695",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction specify the same target text, \"him\", with no change in wording (only formatting/italics). This captures the core intended change (italicizing \"him\") accurately.\n\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing \"authorize\" with \"authorise\". The core intended change (American spelling to British spelling) is captured identically with the same original and corrected text. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to",
"corrected_text": "now, to",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:14.382683",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:23:41.021896",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"For Mr. Pancks\" to \"For, Mr. Pancks\", inserting a comma after \"For\". The prediction changes \"For Mr.\" to \"For, Mr.\", which applies the same core punctuation change (adding a comma after \"For\"). Although the predicted edit includes less surrounding context (it omits \"Pancks\"), it still captures the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"quickly that\") and the same corrected text (\"quickly, that\"), capturing the identical core change\u2014adding a comma after \"quickly\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:14.793310",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-25T23:41:36.858372",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction likewise inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it stops at \"Mr.\" rather than including \"Pancks\"), it captures the same essential punctuation insertion at the correct location. Therefore, the core intended edit matches."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". The core change is identical with no deviation in wording. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_openai_gpt_5_5_pro",
"date": "2026-04-26T00:25:18.388974",
"annotator": "GPT-5.5 Pro (OR)",
"annotation_date": "2026-04-26T00:01:03.227599",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after the word \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The prediction also inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the predicted span is shorter and omits \"Pancks,\" it still captures the core intended punctuation change in the correct location. Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"quickly that\") and the same corrected text (\"quickly, that\"), capturing the identical core change\u2014adding a comma after \"quickly.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2026-04-24",
"release_source": "https://openrouter.ai/openai/gpt-5.5-pro",
"release_notes": "OpenRouter listing for GPT-5.5 Pro. OpenAI describes GPT-5.5 Pro as a higher-compute variant of GPT-5.5."
},
{
"model_name": "GPT-5.6 Luna",
"model_id": "gpt-5.6-luna",
"date": "2026-07-23T15:27:56.402483",
"shots": 2,
"config": {
"model_id": "gpt-5.6-luna",
"display_name": "GPT-5.6 Luna",
"shots": 2,
"temperature": 0.0,
"reasoning_effort": "high",
"date": "2026-07-23",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:23:47.339795",
"file_results": [
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:24.400273",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:23:47.339795",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same core change: adding a period to the end of \"Fellow Travellers\" to make it \"Fellow Travellers.\" The original and corrected texts match the ground truth exactly.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" (\"pockets and\" \u2192 \"pockets, and\"). The original and corrected text strings are identical between ground truth and prediction, capturing the intended punctuation correction precisely."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:28.661519",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:35:39.405291",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\nText Content Accuracy: Both specify the same core change: adding a period to the end of \"Fellow Travellers\" (\"Fellow Travellers\" \u2192 \"Fellow Travellers.\"). The original and corrected texts align perfectly with the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder",
"corrected_text": "yonder,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"yonder\" in the phrase \"yonder to-day\" \u2192 \"yonder, to-day\". The prediction captures this core change by editing \"yonder\" \u2192 \"yonder,\". Although it omits the following word \"to-day\", it still represents the same essential punctuation insertion at the correct location, which is acceptable under the guidelines."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets",
"corrected_text": "pockets,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after the word \"pockets\" in the phrase \"pockets and\" (making it \"pockets, and\"). The prediction also inserts a comma after \"pockets\" (\"pockets\" \u2192 \"pockets,\"). Although it includes less surrounding context (it doesn\u2019t show \"and\"), it captures the core intended punctuation change at the correct location. Therefore, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "sir",
"corrected_text": "Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:27.701300",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:36:37.617640",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. Text Content Accuracy: Both specify the same original_text (\"howling over\") and the same corrected_text (\"howling, over\"), capturing the identical core change of inserting a comma after \"howling\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"pockets and\" to \"pockets, and\", inserting a comma after \"pockets\". The core punctuation correction is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day sir",
"corrected_text": "to-day, sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:23.741257",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:23:26.747142",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same change from \"No sir\" to \"No, sir,\" inserting a comma after \"No.\" The predicted edit captures the complete core change with identical original and corrected text. Therefore, it matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase (\"why did he dine\" \u2192 \"Why did he dine\"). The predicted edit (\"why\" \u2192 \"Why\") captures the core intended change\u2014capitalizing \"why\" at the start. Although it provides less surrounding context, it still correctly represents the essential correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones.",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:19.004310",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:35:51.996762",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"No sir\" to \"No, sir\", inserting a comma after \"No\". The core intended punctuation correction is captured with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase, changing \"why did he dine\" to \"Why did he dine\". The predicted edit captures this core change by specifying \"why\" \u2192 \"Why\". Although it includes less surrounding context than the ground truth, it correctly represents the essential capitalization correction intended."
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:20.816502",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:36:33.842585",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits change \"No sir\" to \"No, sir\" by inserting a comma after \"No\", which captures the core intended correction exactly. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir,",
"corrected_text": "sir;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "reflection,",
"corrected_text": "reflection;",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:23.068117",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:23:03.908697",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial word in the phrase \"if he\" \u2192 \"If he\". The prediction captures the same core change by capitalizing \"if\" \u2192 \"If\". Although it provides less surrounding context (omitting \"he\"), the essential correction\u2014capitalizing the word \"if\"\u2014is accurately represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes the verb from \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The predicted edit captures this core substitution by replacing \"said\" with \"asked\". Although it omits the surrounding context (\"Clennam\"), it still reflects the essential intended change.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:30.815966",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:34:49.362198",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"if he\" to \"If he,\" which fundamentally consists of capitalizing the word \"if\" at the start of the phrase; \"he\" remains unchanged. The prediction captures this core change by converting \"if\" to \"If\" (with less surrounding context). Omitting \"he\" does not alter the essential intended correction, since the only modification is the capitalization of \"if.\""
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" within the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction replaces \"said\" with \"asked\" without including the following context (\"Clennam\"), but it captures the core intended change (\"said\" \u2192 \"asked\"). Including less surrounding text is acceptable as long as the essential substitution is correct."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:22.388697",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:35:10.598864",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to capitalize the initial word in the phrase \"if he\" \u2192 \"If he\". The predicted edit \"if\" \u2192 \"If\" captures the same core correction (capitalizing \u201cif\u201d). The omission of the following word \u201che\u201d is just reduced context and does not alter the essential intended change.\n\nTherefore, the predicted edit correctly captures the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes the wording from \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the same core substitution (\"said\" \u2192 \"asked\") even though it provides less surrounding context (it omits \"Clennam\" from the original/corrected text fields). Since the essential intended change is the verb replacement, the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:14.780709",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:23:54.013791",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"high road\" and the same corrected text \"high-road\", capturing the identical core change (inserting a hyphen). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original span (\"materials costing\") and the same corrected span (\"materials, costing\"), capturing the identical core change\u2014adding a comma after \"materials\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction change \"about and\" to \"about, and\" by inserting a comma after \"about.\" This captures the exact core change with identical original and corrected text.\n\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"earnestly telling\" to \"earnestly, telling\", inserting a comma after \"earnestly\". The core punctuation correction is identical.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change the phrase from \"said when\" to \"said, when\" by inserting a comma after \"said.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"said\" (\"said\" \u2192 \"said,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change, inserting a comma after \"Clennam\" (\"Clennam in\" \u001a \"Clennam, in\"). The original and corrected text strings are identical between ground truth and prediction, capturing the core intended punctuation correction."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:22.895809",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:35:48.473243",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same core change: replacing \"high road\" with \"high-road\" (inserting a hyphen). The original and corrected texts match the ground truth intention exactly.\n\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"materials costing\" to \"materials, costing,\" inserting a comma after \"materials.\" This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "tossed about and",
"corrected_text": "tossed about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"about\" in the phrase \"about and\" \u2192 \"about, and\". The predicted edit applies the same essential punctuation change, just with additional preceding context (\"tossed\"): \"tossed about and\" \u2192 \"tossed about, and\". This captures the core intended correction (adding the comma after \"about\")."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"earnestly telling\" to \"earnestly, telling\", inserting a comma after \"earnestly\". The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "always said when",
"corrected_text": "always said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"said\" in the sequence \"said when\" \u2192 \"said, when.\" The prediction performs the same core punctuation change (inserting a comma after \"said\") but includes additional surrounding context (\"always\"). This still captures the essential intended edit accurately."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "and said",
"corrected_text": "and said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The prediction applies the same core change\u2014adding a comma after \"said\"\u2014but includes additional left-context (\"and said\" \u2192 \"and said,\"). Since the essential modification (inserting a comma immediately after \"said\") is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same corrected text (\"Clennam, in\"), capturing the identical core change of inserting a comma after \"Clennam\". No discrepancy in the essential edit."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:25.781034",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:36:09.772477",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road\", capturing the identical core change (inserting a hyphen). No discrepancies in original or corrected text are present.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \u201cmaterials\u201d in the phrase \u201cmaterials costing\u201d \u2192 \u201cmaterials, costing.\u201d The predicted edit \u201cmaterials\u201d \u2192 \u201cmaterials,\u201d captures the same core punctuation insertion at the correct word boundary. Although it omits the following word \u201ccosting,\u201d it still reflects the essential intended change (adding the comma after \u201cmaterials\u201d)."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"about\" in the phrase \"about and\" \u2192 \"about, and\". The prediction captures the same core punctuation change by editing \"about\" \u2192 \"about,\". Although it omits the surrounding word \"and\" from the original/corrected strings, it still implements the essential intended correction (adding a comma after \"about\")."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly",
"corrected_text": "earnestly,",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after the word \"earnestly\" in the phrase \"earnestly telling\" (becoming \"earnestly, telling\"). The prediction captures the same core punctuation insertion by changing \"earnestly\" to \"earnestly,\". Although it provides less surrounding context than the ground truth, it implements the identical essential change (comma insertion after \"earnestly\")."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth edit inserts a comma after the word \"said\" in the phrase \"said when\" \u2192 \"said, when\". The predicted edit \"said\" \u2192 \"said,\" captures the same core change (adding a comma immediately after \"said\"). Although it includes less surrounding context (it omits \"when\"), it still reflects the essential punctuation correction intended by the ground truth."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"said\" to \"said,\" by adding a comma after the word. The core intended punctuation change is identical.\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is the insertion of a comma immediately after \"Clennam\" in the phrase \"Clennam in\" \u2192 \"Clennam, in.\" The prediction captures the same core punctuation change (\"Clennam\" \u2192 \"Clennam,\") even though it includes less surrounding context (it omits the following word \"in\"). Since the essential intended edit\u2014adding a comma after \"Clennam\"\u2014is correctly represented, the content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association",
"corrected_text": "association,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:25:58.557930",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:23:05.718122",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:00.465163",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:34:47.277940",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:01.777755",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:35:10.377344",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:17.293537",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:24:14.557993",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Let Loose\" to become \"Let Loose.\" The original_text and corrected_text are identical to the ground truth, capturing the intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The original and corrected texts are identical between ground truth and prediction, so the core change is captured exactly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"replacement\", so the type matches exactly.\n2. Text Content Accuracy: Both perform the same core change, replacing \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The original_text and corrected_text match the ground truth exactly, capturing the intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change: inserting a comma after \"Chalons\" in the phrase \"Chalons was\" to become \"Chalons, was\". The original and corrected texts are identical between ground truth and prediction, so the core change is captured exactly."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream like",
"corrected_text": "stream, like",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:26.569904",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:35:59.906282",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Let Loose\" to \"Let Loose.\" by adding a period at the end. The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after the word \"man\" in the phrase \"man slowly\" (resulting in \"man, slowly\"). The predicted edit changes \"man\" to \"man,\", which captures the same core punctuation insertion at the correct location (after \"man\"). Although it omits the following word \"slowly\" from the span, it still represents the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream",
"corrected_text": "stream,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons,",
"corrected_text": "Chalons",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:23.311665",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:36:38.468956",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"Let Loose\" to \"Let Loose.\" (adding a period). The predicted edit captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the same original text \"Saone\" with the same corrected text \"Sa\u00f4ne\" (adding the circumflex over the o). The core change is identical and fully captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both edits change \"man slowly\" to \"man, slowly\" by inserting a comma after \"man,\" which captures the core intended correction precisely. Therefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the core change of inserting a comma after \"Chalons\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream",
"corrected_text": "stream,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:30.190783",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:23:54.821624",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"heart\" in the phrase \"heart so\" to make it \"heart, so\". The original_text and corrected_text match exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "that she thanked",
"corrected_text": "that she indeed thanked",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart (",
"corrected_text": "heart, (",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling;",
"corrected_text": "smiling,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:33.406288",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:35:52.310583",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same change from \"heart so\" to \"heart, so\", correctly capturing the core intended punctuation insertion (adding a comma after \"heart\"). No discrepancies in the original or corrected text content."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit is an \"insertion,\" and the predicted edit is also an \"insertion,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert the word \"indeed\" into the phrase, turning \"as she did\" into \"as indeed she did.\" The predicted edit inserts \"indeed\" (even though it leaves original_text empty and does not provide surrounding context). This captures the core intended change\u2014the addition of \"indeed\"\u2014so the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the verb phrase from \"I hope so\" to \"I hoped so\"\u2014the essential change is \"hope\" \u2192 \"hoped\" (present to past tense). The prediction captures this core change by replacing \"hope\" with \"hoped\". Although it omits surrounding context (\"I\" and \"so\"), it still correctly represents the intended textual modification."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling; \u201cwill",
"corrected_text": "smiling, \u201cwill",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:26:33.235218",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:36:03.265664",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"heart so\" to \"heart, so,\" inserting a comma after \"heart.\" This captures the core intended correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "did, if",
"corrected_text": "did, indeed, if",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling;",
"corrected_text": "smiling,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:05.181527",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:24:04.419759",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly. Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"hers\" in the phrase \"hers and,\" yielding \"hers, and.\" The original and corrected text strings match exactly, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"hers matters\" to \"hers, matters,\" inserting a comma after \"hers.\" The core change is identical, with no missing or conflicting modifications."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), inserting a comma after \"moment.\" This captures the core intended punctuation change with no discrepancies."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction apply the same core change: inserting a comma after \"beautiful\" in the phrase \"beautiful that,\" resulting in \"beautiful, that.\" The original and corrected text strings match exactly, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change from \"they\" to \"They\" with no differences in original or corrected text. Therefore, the predicted edit fully captures the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:05.429521",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:36:00.855863",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: \"hers and\" \u2192 \"hers, and\" (inserting a comma after \"hers\"). This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction change \"hers matters\" to \"hers, matters\" by inserting a comma after \"hers.\" This captures the core intended correction with identical original and corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both edits change the same phrase from \"moment with\" to \"moment, with\", inserting a comma after \"moment\". The core punctuation change is identical. Therefore, the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original phrase \"beautiful that\" and the same correction \"beautiful, that\", inserting a comma after \"beautiful\". This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth.\n2) Text Content Accuracy: Both ground truth and prediction change \"they\" to \"They,\" capturing the identical core capitalization correction with no differences in the original or corrected text. Therefore, the prediction matches the ground truth\u2019s intended edit."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:11.951503",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:36:10.104823",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" in the phrase \"hers and\" to make it \"hers, and\". The prediction edits \"hers\" to \"hers,\", which captures the same core punctuation change (adding a comma immediately after \"hers\"). Although the prediction includes less surrounding context (it omits \"and\"), it still represents the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is the insertion of a comma after the word \"hers\" in the phrase \"hers matters\" \u2192 \"hers, matters\". The predicted edit \"hers\" \u2192 \"hers,\" captures the same essential punctuation insertion (adding a comma immediately after \"hers\"). Although it omits the following word \"matters\" as context, it still reflects the core intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"moment\" (\"moment with\" \u2192 \"moment, with\"). The prediction captures the same core punctuation change by editing \"moment\" \u2192 \"moment,\". Although it omits the following word \"with\" from the span, it still represents the essential intended correction (adding the comma after \"moment\")."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful",
"corrected_text": "beautiful,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"beautiful\" (\"beautiful that\" \u2192 \"beautiful, that\"). The prediction captures the same core punctuation change by editing \"beautiful\" \u2192 \"beautiful,\". Although the prediction does not include the following word \"that\" as context, it still reflects the essential intended correction (adding a comma after \"beautiful\")."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify changing \"they\" to \"They,\" which captures the same core correction with identical original and corrected text. Therefore, the prediction matches the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:22.869344",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:24:13.866307",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"shore\" in the phrase \"shore and\" \u2192 \"shore, and.\" The predicted edit \"shore\" \u2192 \"shore,\" captures the same core punctuation insertion (a comma immediately after \"shore\"). Although it omits the following word \"and\" from the span, it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\". This captures the same core change with no discrepancy. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "that,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy",
"corrected_text": "fancy,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:08.157199",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:36:08.755947",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"Thank you!\" to \"Thank you,\"\u2014replacing the exclamation mark with a comma. The core intended correction is captured identically. Therefore, the predicted edit matches the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original segment (\"shore and\") and the same correction (\"shore, and\"), capturing the identical core change of inserting a comma after \"shore.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the types match exactly. Text Content Accuracy: Both edits replace the same original text (\"that\") with the same corrected text (\"than\"), capturing the identical core change. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:21.759001",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:36:38.893143",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both change the original text \"that\" to the corrected text \"than\". The core intended substitution is captured precisely with no discrepancy. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you",
"corrected_text": "you,",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:33.831876",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:25:12.444476",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\". The original and corrected texts are identical between ground truth and prediction, so the essential edit is captured correctly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted type is \"capitalization,\" which exactly matches the ground truth edit type.\n2. Text Content Accuracy: The predicted change converts \"city\" to \"City,\" which is identical to the ground truth core change. No additional or missing content alters the intended correction.\nTherefore, both criteria are satisfied and the prediction correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same punctuation change from \"them,\" to \"them;\". The core correction (replacing a comma with a semicolon after \"them\") is identical.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The essential edit is identical, with no discrepancy in the modified text."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the types match exactly. Text Content Accuracy: Both ground truth and prediction change \"motionless\" to \"motionless,\" by adding a comma immediately after the word. The core intended change is captured identically. Therefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same core change: adding a comma after \"view\" (\"view\" \u2192 \"view,\"). There is no discrepancy in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the punctuation after the word \"existence\" from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The predicted edit also results in the same corrected form \"existence;\". Although the predicted \"original_text\" omits the comma (\"existence\" instead of \"existence,\"), it still captures the core intended change at that spot: the terminal punctuation should be a semicolon. Therefore, the essential correction is correctly represented."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:42.073504",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:37:07.598892",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2. Text Content Accuracy: Both edits change \"Chapter XXIX\" to \"Chapter XXIX.\" by adding a period at the end. The core intended punctuation change is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "MRS. FLINTWINCH GOES ON DREAMING",
"corrected_text": "MRS. FLINTWINCH GOES ON DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period after the word \"DREAMING\" (\"DREAMING\" \u2192 \"DREAMING.\"). The predicted edit adds a period at the end of a longer phrase that ends with the same word (\"... DREAMING\" \u2192 \"... DREAMING.\"). This includes extra surrounding context but preserves the core intended change: inserting a terminal period after \"DREAMING\". Therefore, the essential edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"capitalization,\" which exactly matches the ground truth type.\nText Content Accuracy: The ground truth changes \"city\" to \"City.\" The prediction makes the same change from \"city\" to \"City\" with no discrepancy. Therefore, it captures the core intended edit exactly."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as type \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the identical core change of adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). There is no divergence in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"view\" (\"view\" \u2192 \"view,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:44.136550",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:38:08.378822",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XXIX",
"corrected_text": "XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period to the end of the chapter heading: \"Chapter XXIX\" \u2192 \"Chapter XXIX.\" The prediction applies the same essential punctuation change to the key element of that heading: \"XXIX\" \u2192 \"XXIX.\" Although it omits the preceding word \"Chapter\" in the original/corrected text span, it still captures the core intended edit\u2014adding a terminal period after \"XXIX\"\u2014which is the actual correction being made.\n\nTherefore, both criteria are satisfied."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: \"DREAMING\" \u2192 \"DREAMING.\" (adding a period). This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both indicate the same core change: \"city\" \u2192 \"City\". The corrected text matches the ground truth\u2019s intended capitalization change with no discrepancy.\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both indicate replacing the original text \"of\" with the corrected text \"to\". The core change is identical, with no discrepancy in the words being replaced or the intended correction. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). There is no discrepancy in the original or corrected text content. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege-lord",
"corrected_text": "liege-lord,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:35.316730",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:25:15.032849",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth specifies a \"replacement\" edit type, and the prediction also uses \"replacement\", so the type matches exactly.\nText Content Accuracy: Both ground truth and prediction replace \"authorize\" with \"authorise\". The core change (American spelling to British spelling) is captured identically with no discrepancies. Therefore, the predicted edit matches the intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ";",
"corrected_text": ",",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a little",
"corrected_text": "a little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ",",
"corrected_text": "",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:35.995507",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:37:51.102776",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\n2) Text Content Accuracy: Both identify the same target text \"him\" and the corrected text remains \"him\" with the implied change being formatting (italics). The prediction captures the same core formatting change on the same word. Therefore, it matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both change the same original text \"authorize\" to the same corrected text \"authorise\", capturing the identical core spelling change (American to British spelling). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a little",
"corrected_text": "a little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "boiling-over",
"corrected_text": "boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it.",
"corrected_text": "it,",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "of",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:49.168170",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:37:39.856341",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" to \"driest\" and inserts \"of\" to make the phrase \"driest of snorts\" (within the larger context \"of the ... .\"). The prediction likewise replaces \"dryest snorts\" with \"driest of snorts,\" capturing the same essential spelling correction and the insertion of \"of.\" Although the prediction omits surrounding context (\"of the\" and the final period), it still reflects the core intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing \"authorize\" with \"authorise\". The core change (American spelling to British spelling) is identical. Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "little premature",
"corrected_text": "little too premature",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "score;",
"corrected_text": "score,",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:53.281144",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T11:25:38.227339",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "For",
"corrected_text": "",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly,",
"corrected_text": "quickly",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me",
"corrected_text": "me,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:54.852195",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T12:37:39.992086",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For",
"corrected_text": "For,",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2. Text Content Accuracy: The ground truth inserts a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks.\" The prediction captures the same core punctuation change by editing \"For\" \u2192 \"For,\". Although it omits the following context (\"Mr. Pancks\"), it still correctly represents the essential intended edit: adding a comma immediately after \"For.\""
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly,",
"corrected_text": "quickly",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me",
"corrected_text": "me,",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.6-luna",
"date": "2026-07-23T15:27:56.402483",
"annotator": "GPT-5.6 Luna",
"annotation_date": "2026-07-23T13:38:48.765831",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bedroom,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For,",
"corrected_text": "For",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly,",
"corrected_text": "quickly",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me,",
"corrected_text": "me",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2026-07-09",
"release_source": "https://community.openai.com/t/introducing-gpt-5-6-series-sol-terra-and-luna-coming-july-9-10am-pt/1384931",
"release_notes": "Fast/low-cost tier of the GPT-5.6 series."
},
{
"model_name": "GPT-5.6 Sol",
"model_id": "gpt-5.6-sol",
"date": "2026-07-23T15:21:49.132174",
"shots": 2,
"config": {
"model_id": "gpt-5.6-sol",
"display_name": "GPT-5.6 Sol",
"shots": 2,
"temperature": 0.0,
"reasoning_effort": "high",
"date": "2026-07-23",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:18:05.416852",
"file_results": [
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:37.980171",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:18:05.416852",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The core intended punctuation change is captured identically, with no discrepancy in the original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to make it \"pockets, and\". The original and corrected text strings are identical between ground truth and prediction, so the intended punctuation change is captured correctly."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day Sir",
"corrected_text": "to-day, Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:39.531125",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:29:59.088088",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change: inserting a comma after \"howling\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"pockets and\" to \"pockets, and,\" correctly capturing the intended insertion of a comma after \"pockets.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "TRAVELLERS",
"corrected_text": "TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day Sir",
"corrected_text": "to-day, Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:33.548762",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:30:34.392714",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"pockets\" in the phrase \"pockets and\" to become \"pockets, and.\" The original and corrected text match the ground truth precisely.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "FELLOW TRAVELLERS",
"corrected_text": "FELLOW TRAVELLERS.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day Sir",
"corrected_text": "to-day, Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:43.060091",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:18:12.374780",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"No sir\" to \"No, sir\", correctly inserting a comma after \"No\". The core intended punctuation correction is captured with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the initial \"why\" in the phrase \"why did he dine\" to \"Why\" (capitalizing the first word). The prediction captures the same core change by editing \"why\" \u2192 \"Why\". Although it includes less surrounding context, it correctly represents the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "!",
"corrected_text": "?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the ending punctuation in \"cold stones!\" to \"cold stones?\" (i.e., replacing \"!\" with \"?\"). The prediction represents the same core change by specifying \"!\" \u2192 \"?\" without the surrounding words. This still correctly captures the essential intended correction (exclamation mark to question mark)."
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:35.757438",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:29:46.412548",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"No sir\") and the same correction (\"No, sir\"), inserting a comma after \"No\". The core intended punctuation change is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction use the type \"capitalization\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is capitalizing the initial word in the phrase (\"why did he dine\" \u2192 \"Why did he dine\"), i.e., changing \"why\" to \"Why\" at the start of the sentence. The predicted edit captures this core change directly (\"why\" \u2192 \"Why\"), even though it includes less surrounding context. This still reflects the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation mark to a question mark (\"cold stones!\" \u2192 \"cold stones?\"). The prediction captures the same core change (\"stones!\" \u2192 \"stones?\") but with less surrounding context. Since the essential intended correction is the replacement of \"!\" with \"?\" after the same word sequence ending in \"stones\", the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:33.539477",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:30:44.743368",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth. Text Content Accuracy: Both ground truth and prediction change \"No sir\" to \"No, sir,\" inserting a comma after \"No.\" This captures the exact core correction with identical original and corrected text. Therefore, the prediction correctly matches the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial \"w\" in \"why did he dine\" (changing \"why\" to \"Why\" while leaving the rest unchanged). The prediction captures this same core change by editing \"why\" \u2192 \"Why\". Although the prediction includes less surrounding context than the ground truth, it correctly reflects the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones\u201d",
"corrected_text": "stones?\u201d",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:34.504897",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:17:57.822504",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the word \"if\" in the phrase \"if he\" (\"if he\" \u2192 \"If he\"). The prediction likewise changes \"if\" \u2192 \"If\". Although it includes less surrounding context (omitting \"he\"), it captures the core intended change: capitalizing \"if.\" Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes the word \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the same core substitution by replacing \"said\" \u2192 \"asked\" (omitting the unchanged context \"Clennam\"). Since the essential intended change is the same word replacement, the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:34.831316",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:29:37.936070",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial \"if\" in the phrase \"if he\" \u2192 \"If he.\" The prediction applies the same core change to the word \"if\" \u2192 \"If\" without including the following word \"he.\" This is acceptable because it captures the essential intended change (capitalizing the initial 'i' in 'if'); the omitted context does not alter the substance of the correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth intends to change the word \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam.\" The prediction captures the same core substitution (\"said\" \u2192 \"asked\"), even though it omits the surrounding context (\"Clennam\"). Omitting unchanged context is acceptable because the essential edit (replacing \"said\" with \"asked\") is correctly represented."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:36.899437",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:30:16.922437",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase \"if he\" \u2192 \"If he.\" The prediction captures the core intended change by capitalizing \"if\" \u2192 \"If.\" Although it omits the following word \"he\" from the original/corrected text span, it still represents the same essential capitalization correction. Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The predicted edit replaces \"said\" \u2192 \"asked\" without including the following context \"Clennam\". This still captures the core intended change (changing the verb from \"said\" to \"asked\"), and omission of surrounding context is acceptable under the criteria.\n\nTherefore, the predicted edit matches the ground truth's intention."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:09.004829",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:18:30.690620",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: Both edits change \"high road\" to \"high-road\", capturing the same core correction (adding a hyphen). There is no mismatch in the original or corrected text. Therefore, the predicted edit accurately reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change\u2014adding a comma after \"materials.\" Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same change from \"about and\" to \"about, and,\" inserting a comma after \"about.\" This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change\u2014adding a comma after \"earnestly\" in the phrase \"earnestly telling\" to become \"earnestly, telling\". The original and corrected texts are identical between ground truth and prediction, so the core change is captured correctly."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction apply the same core change, inserting a comma between \"said\" and \"when\" (\"said when\" \u001a \"said, when\"). The original and corrected text strings match exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a comma after \"said\" (\"said\" \u2192 \"said,\"). There is no discrepancy in original or corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Clennam in\" to \"Clennam, in\" by inserting a comma after \"Clennam\". The original and corrected strings match the ground truth\u2019s core change exactly. Therefore, the predicted edit correctly captures the intended correction."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:18.377184",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:30:47.785804",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"materials\" in the phrase \"materials costing\" \u2192 \"materials, costing\". The predicted edit \"materials\" \u2192 \"materials,\" captures this core punctuation change (adding the comma after \"materials\"). Although the prediction includes less surrounding context (it omits \"costing\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"about\" in the phrase \"about and\" \u2192 \"about, and.\" The predicted edit \"about\" \u2192 \"about,\" captures the same essential punctuation insertion (a comma immediately after \"about\"). Although the prediction includes less surrounding context (it omits \"and\"), it still represents the core intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association,",
"corrected_text": "association",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to remove the comma after \"association\" so the phrase reads \"association did\" instead of \"association, did.\" The predicted edit removes the comma after \"association\" (\"association,\" \u2192 \"association\"), which captures the same core punctuation change. Although the prediction includes less surrounding context (it omits \"did\"), it still implements the essential intended correction: deleting the comma after \"association.\""
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly",
"corrected_text": "earnestly,",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"earnestly\" in the phrase \"earnestly telling\" (i.e., \"earnestly, telling\"). The prediction captures the same core change by converting \"earnestly\" to \"earnestly,\". Although it omits the following word \"telling\" from the original/corrected span, it still correctly represents the essential punctuation insertion at the correct location (after \"earnestly\"). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, adding a comma after \"said\" (\"said\" \u2192 \"said,\"). There is no discrepancy in original or corrected text. Therefore the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said,",
"corrected_text": "said",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam,",
"corrected_text": "Clennam",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:27.924844",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:31:16.136437",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road\" (adding a hyphen between the words). The core change is identical, with no deviation in original or corrected text.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change\u2014adding a comma after \"materials\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma to change \"about and\" to \"about, and\". There is no discrepancy in original or corrected text. Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association,",
"corrected_text": "association",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth removes the comma after \u201cassociation\u201d in the phrase \u201cassociation, did\u201d to make it \u201cassociation did.\u201d The predicted edit also removes the comma after \u201cassociation\u201d (\u201cassociation,\u201d \u2192 \u201cassociation\u201d). Although the prediction includes less surrounding context (it omits \u201cdid\u201d), it captures the core intended change: deleting the comma after \u201cassociation.\u201d Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"earnestly telling\" to \"earnestly, telling,\" inserting a comma after \"earnestly.\" This captures the core intended punctuation correction with identical original and corrected text.\n\nTherefore, the prediction correctly matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"said when\") and the same corrected text (\"said, when\"), capturing the identical core change of inserting a comma after \"said.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same corrected text (\"Clennam, in\"), capturing the identical core change: insertion of a comma after \"Clennam.\" Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Why bless",
"corrected_text": "Why, bless",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said;",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:19:59.533819",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:17:51.944558",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW",
"corrected_text": "SUN AND SHADOW.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:03.930239",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:29:32.673398",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "SUN AND SHADOW",
"corrected_text": "SUN AND SHADOW.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:06.837640",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:30:12.242698",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly. Text Content Accuracy: Both ground truth and prediction change \"Sun and Shadow\" to \"Sun and Shadow.\" by adding a period at the end. The core intended punctuation change is captured exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:21.503342",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:19:04.285832",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both replace the original text \"Saone\" with the corrected text \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"man\" in the phrase \"man slowly\" \u2192 \"man, slowly\". The predicted edit \"man\" \u2192 \"man,\" captures the same essential punctuation insertion (a comma immediately after \"man\"). Although it omits the following word \"slowly\" as context, it still reflects the core intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"Chalons\" in the phrase \"Chalons was\" \u2192 \"Chalons, was\". The prediction captures the same core change by editing \"Chalons\" \u2192 \"Chalons,\". Although the prediction includes less surrounding context (it omits \"was\"), it still reflects the essential intended punctuation insertion at the correct location (after \"Chalons\")."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream",
"corrected_text": "stream,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:31.766076",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:30:30.159509",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit type matches exactly: both are a \"replacement\". Text content also matches the intended correction: both replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the o). Therefore, the predicted edit fully captures the ground truth change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both perform the same core change, replacing \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The original_text and corrected_text are identical between ground truth and prediction, so the intended correction is captured precisely."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"man\" in the phrase \"man slowly\" \u2192 \"man, slowly.\" The predicted edit \"man\" \u2192 \"man,\" captures the same essential punctuation insertion (a comma immediately after \"man\"). Although it includes less surrounding context (doesn\u2019t show \"slowly\"), it still represents the core intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"Chalons was\" to \"Chalons, was\", i.e., it inserts a comma immediately after \"Chalons\". The prediction changes \"Chalons\" to \"Chalons,\", which captures the same core punctuation insertion at the same location. The fact that the prediction omits the following word \"was\" is acceptable because it still represents the essential intended change (adding the comma after \"Chalons\")."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "LET LOOSE",
"corrected_text": "LET LOOSE.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream",
"corrected_text": "stream,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:33.005015",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:30:50.178281",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"Let Loose\" to \"Let Loose.\"\u2014adding a period at the end. The core intended change is captured precisely with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2. Text Content Accuracy: Both specify the same original text \"Saone\" and the same corrected text \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core change is identical and fully captured."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"man\" in the phrase \"man slowly\" \u2192 \"man, slowly.\" The predicted edit \"man\" \u2192 \"man,\" captures the same core punctuation insertion at the correct location (after \"man\"). Although it includes less surrounding context (it omits \"slowly\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). The prediction likewise represents adding a comma after \"Chalons\" (\"Chalons\" \u2192 \"Chalons,\"). Although it includes less surrounding context (it omits the following word \"was\"), it captures the core intended punctuation change in the correct location. Therefore, the essential edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XI",
"corrected_text": "Chapter XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stream",
"corrected_text": "stream,",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:40.130004",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:18:53.037097",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same core change, inserting a comma after \"heart\" (\"heart so\" \u2192 \"heart, so\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous,",
"corrected_text": "generous",
"observed_line_number": 22,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is removing the comma after \"generous\" in the phrase \"generous, Mr.\" \u2192 \"generous Mr.\" The predicted edit \"generous,\" \u2192 \"generous\" captures the same core punctuation change (deleting that comma). Although it includes less surrounding context (it omits \"Mr.\"), it still correctly reflects the essential intended edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "as she",
"corrected_text": "as indeed she",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches.\n\n2) Text Content Accuracy: The ground truth change is inserting the word \"indeed\" into the phrase, turning \"as she did\" into \"as indeed she did.\" The prediction captures the same core insertion\u2014adding \"indeed\" after \"as\"\u2014shown as \"as she\" \u2192 \"as indeed she.\" Although the prediction provides a shorter surrounding span (omitting the trailing \"did\"), it still represents the essential intended correction: insertion of \"indeed\" in that location. Therefore, the text-content intention is correctly captured."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes the verb tense in the phrase \"I hope so\" \u2192 \"I hoped so\" (i.e., \"hope\" \u2192 \"hoped\"). The prediction captures this core change directly as \"hope\" \u2192 \"hoped\". Although it omits surrounding context (\"I\" and \"so\"), the essential intended modification is correctly represented."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam smiling",
"corrected_text": "Clennam, smiling",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:38.705217",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:30:50.733213",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both ground truth and prediction change \"heart so\" to \"heart, so\", inserting a comma after \"heart\". This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": 22,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: removing the comma after \"generous\" in \"generous, Mr.\" to make it \"generous Mr.\". The original_text and corrected_text are identical between ground truth and prediction, capturing the core intended punctuation change."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness),",
"corrected_text": "ness);",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam smiling",
"corrected_text": "Clennam, smiling",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "with some hurried words to the effect that",
"corrected_text": "indeed",
"observed_line_number": 26,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:37.291613",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:32:39.247125",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"heart so\" to \"heart, so,\" inserting a comma after \"heart.\" The core change is identical with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \u201clength\u201d and \u201csaid\u201d (\"length said\" \u2192 \"length, said\"). The prediction does the same essential punctuation insertion, but includes extra surrounding context (\u201cat\u201d): \"at length said\" \u2192 \"at length, said\". Since the core intended change (adding the comma after \u201clength\u201d) is correctly captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous Mr.",
"corrected_text": "generous, Mr.",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "she thanked",
"corrected_text": "she indeed thanked",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "ness) she",
"corrected_text": "ness), she",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:43.912878",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:18:35.724596",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"hers and\") and the same corrected text (\"hers, and\"), capturing the identical core change\u2014adding a comma after \"hers.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have the type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"hers matters\" to \"hers, matters\", inserting a comma after \"hers\". The core change is identical, so the text content matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"moment with\" to \"moment, with\" by inserting a comma after \"moment.\" The original and corrected text match the ground truth\u2019s core change exactly.\nTherefore, the predicted edit correctly captures the intended correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same corrected text (\"beautiful, that\"), inserting a comma after \"beautiful.\" This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change the same word from \"they\" to \"They\". The core change (capitalizing the initial letter) is captured identically. Therefore, the prediction matches the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:56.714335",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:30:23.848328",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth. Text Content Accuracy: Both ground truth and prediction change \"hers and\" to \"hers, and\" by inserting a comma after \"hers.\" The core punctuation change is identical, with matching original and corrected text. Therefore, the prediction correctly captures the intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"hers matters\" to \"hers, matters\", inserting a comma after \"hers\". The essential punctuation correction is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change of inserting a comma after \"moment\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"beautiful that\" to \"beautiful, that\", inserting a comma after \"beautiful\". The original and corrected text match the ground truth\u2019s core change exactly.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth.\n2) Text Content Accuracy: Both ground truth and prediction change the word \"they\" to \"They,\" capturing the same core capitalization correction with no discrepancy in original or corrected text. Therefore, the predicted edit fully matches the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:20:59.941212",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:31:15.859961",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"hers\" in the phrase \"hers and\" to make it \"hers, and\". The core punctuation modification is identical in original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma between \"hers\" and \"matters\" (\"hers matters\" \u0012 \"hers, matters\"). The core correction is identical, with no discrepancies in original or corrected text.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"moment with\" to \"moment, with\", correctly inserting a comma after \"moment\". The core change and surrounding text match the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both ground truth and prediction change \"beautiful that\" to \"beautiful, that\", inserting a comma after \"beautiful\". The core punctuation change is identical, with no discrepancy in the original or corrected text. Therefore, the prediction captures the intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change from \"they\" to \"They\" with identical original and corrected text. Therefore, the predicted edit fully captures the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:06.662363",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:19:17.204832",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same core change from \"Thank you!\" to \"Thank you,\" (replacing the exclamation point with a comma). The predicted edit exactly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both indicate replacing the original text \"that\" with the corrected text \"than\". The core change is identical, with no discrepancies in the words being replaced or the intended correction. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:14.524838",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:30:45.338270",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly. Text Content Accuracy: The predicted change matches the ground truth core correction, replacing the exclamation mark with a comma (\"Thank you!\" \u2192 \"Thank you,\"). No discrepancies in the original or corrected text. Therefore, the prediction correctly captures the intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same correction (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\nText Content Accuracy: Both replace the original text \"that\" with the corrected text \"than\". The core change is identical, with no discrepancy in the words being replaced or the intended correction.\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:13.759083",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:31:23.220163",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original segment \"that, in\" and the same correction \"that in,\" i.e., removing the comma. The core intended change is captured exactly with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation: removing the comma in \"fancy, whatever\" to yield \"fancy whatever.\" The core punctuation change is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core punctuation change: \"Thank you!\" is corrected to \"Thank you,\" (replacing the exclamation point with a comma). The original and corrected texts match the ground truth exactly, so the intended edit is captured correctly."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change\u2014adding a comma after \"shore\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the types match exactly. Text Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\". The core intended change is identical in the prediction and the ground truth. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:27.783557",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:20:07.317983",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both identify the same core change: converting \"city\" to \"City\". There is no difference in original or corrected text content. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both replace the original text \"of\" with the corrected text \"to\". The core intended change is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"them,\" to \"them;\". The core intended punctuation change (comma to semicolon) is captured precisely with identical original and corrected text.\nTherefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit applies the same core change, adding a comma after \"existence\" within a longer phrase (\"existence at\" \u2192 \"existence, at\"). This includes extra surrounding context but preserves the essential punctuation insertion at the same point. \u0644\u0630\u0644\u0643 the prediction captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core punctuation change from \"it,\" to \"it;\" with identical original and corrected text. Therefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The prediction applies the same core change within a slightly larger span of text (\"motionless when\" \u2192 \"motionless, when\"), which correctly captures the intended comma insertion after \"motionless\". The extra context (including \"when\") does not alter the essential edit.\n\nTherefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text \"stand-still,\" and the same corrected text \"stand-still;\", capturing the identical core change (comma to semicolon). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit adds a comma in the same location within a longer snippet (\"view by\" \u2192 \"view, by\"). This includes additional context but captures the same essential punctuation insertion immediately after \"view.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "xxix",
"corrected_text": "xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence;",
"corrected_text": "existence,",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:49.132174",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:31:41.866369",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change \"DREAMING\" to \"DREAMING.\", adding a period at the end. This captures the same core punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same core change from \"city\" to \"City\" with no differences in original or corrected text. Therefore, the prediction fully captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"them,\" to \"them;\". The core punctuation substitution (comma to semicolon) is identical, so the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence at",
"corrected_text": "existence, at",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). The predicted edit applies the same core punctuation change, adding a comma immediately after \"existence\" within a slightly larger context (\"existence at\" \u2192 \"existence, at\"). This preserves the essential intended correction.\n\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless when",
"corrected_text": "motionless, when",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit applies the same core punctuation change by converting \"motionless when\" to \"motionless, when,\" which still places the comma immediately after \"motionless.\" The extra surrounding word (\"when\") is allowable context and does not alter the essential intended correction.\n\nTherefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth edit inserts a comma after the word \"view\" (\"view\" \u2192 \"view,\"). The predicted edit also inserts a comma immediately after \"view\" (\"view by\" \u2192 \"view, by\"). Although the prediction includes additional surrounding context (the following word \"by\"), it captures the same core change: adding a comma after \"view\"."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "each",
"corrected_text": "to each",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "contracted existence",
"corrected_text": "contracted existence;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "lord",
"corrected_text": "Lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:48.775212",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:32:01.558017",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both identify the same change from \"city\" to \"City\". The core correction (capitalizing the first letter) is captured precisely with no discrepancy. Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\nText Content Accuracy: Both identify the same substitution, changing \"of\" to \"to\". The core change is identical with no deviation.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after the word \"existence\" (\"existence\" \u2192 \"existence,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same core change: adding a comma after \"motionless\" (\"motionless\" \u2192 \"motionless,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a comma after \"view\" (\"view\" \u2192 \"view,\"). The corrected text matches the ground truth precisely.\nTherefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to make the punctuation following the word \"existence\" a semicolon (replacing the original comma). The prediction also results in \"existence;\". Although the predicted \"original_text\" omits the comma (\"existence\" instead of \"existence,\"), it still captures the core intended correction: the punctuation after \"existence\" should be a semicolon. Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:33.213543",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:19:39.331762",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\nText Content Accuracy: Both edits apply italics to the same word, with original_text \"him\" and corrected_text \"him\" (indicating a styling change only). The core intended change\u2014italicizing \"him\"\u2014is captured identically."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"now I\" to \"now, I\", inserting a comma after \"now\". The core correction is identical. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"replacement\", so the type matches exactly.\n2) Text Content Accuracy: Both indicate replacing \"authorize\" with \"authorise\". The core change (American to British spelling) is identical, with no discrepancy in original or corrected text. Therefore the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now though",
"corrected_text": "now, though",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "dryest snorts",
"corrected_text": "dryest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a little",
"corrected_text": "a little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to",
"corrected_text": "now, to",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:41.383739",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:31:33.258305",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\n2) Text Content Accuracy: Both indicate the same target text \"him\" with no change in wording, only formatting via italicization. The core intended change (italicize the word \"him\") is fully captured. Therefore, the predicted edit matches the ground truth intent."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now",
"corrected_text": "now,",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"now\" in the phrase \"now I\" \u2192 \"now, I\". The prediction captures the same core punctuation change by editing \"now\" \u2192 \"now,\". Although it omits the following word (\"I\") in the edit span, it still correctly represents the essential intended correction: adding a comma after \"now\"."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is a \"replacement\", and the predicted edit type is also \"replacement\", so the type matches exactly. Text Content Accuracy: Both specify replacing \"authorize\" with \"authorise\". The core change (American to British spelling) is identical, with no discrepancies in original or corrected text. Therefore, the prediction correctly captures the intended edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now",
"corrected_text": "now,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "of",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a little",
"corrected_text": "a little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now",
"corrected_text": "now,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you",
"corrected_text": "you,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:41.113920",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:32:06.271917",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\n2) Text Content Accuracy: Both identify the same target text \"him\" with no change in wording (only formatting/italicization). The predicted edit captures the same core intention: italicize \"him\".\nTherefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"authorize\" with \"authorise\". The core intended change (American to British spelling) is captured identically with no mismatch in original or corrected text.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam;",
"corrected_text": "Clennam,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a little",
"corrected_text": "a little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to",
"corrected_text": "now, to",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "that",
"corrected_text": "that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:36.037806",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T11:19:43.486737",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks.\" The prediction likewise inserts a comma after \"For\" in \"For Mr.\" \u2192 \"For, Mr.\" Although the predicted span truncates the following context (omitting \"Pancks\"), it still captures the core intended punctuation change: adding a comma after \"For\" before \"Mr.\" Therefore, the essential edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both edits change the phrase from \"quickly that\" to \"quickly, that\" by inserting a comma after \"quickly.\" This captures the same core punctuation change with identical original and corrected text. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "making me not",
"corrected_text": "making me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:39.149820",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T12:31:23.280602",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches.\n\n2) Text Content Accuracy: The ground truth change is the insertion of a comma after \"For\" (\"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\"). The predicted edit also inserts a comma after \"For\" (\"For Mr.\" \u2192 \"For, Mr.\"). Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the same core punctuation change in the same location and thus matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that\", inserting a comma after \"quickly\". The core change is identical, with no discrepancies in the original or corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.6-sol",
"date": "2026-07-23T15:21:39.893651",
"annotator": "GPT-5.6 Sol",
"annotation_date": "2026-07-23T13:31:36.011819",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks.\" The prediction similarly inserts a comma after \"For\" in \"For Mr.\" \u2192 \"For, Mr.\" Although the predicted snippet omits \"Pancks,\" it captures the core intended punctuation change (adding a comma after \"For\" before \"Mr.\"). Therefore, the essential edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change, inserting a comma after \"quickly\" in the phrase \"quickly that\" to become \"quickly, that\". The core punctuation correction is identical. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2026-07-09",
"release_source": "https://community.openai.com/t/introducing-gpt-5-6-series-sol-terra-and-luna-coming-july-9-10am-pt/1384931",
"release_notes": "Flagship tier of the GPT-5.6 series (Sol/Terra/Luna), released 2026-07-09."
},
{
"model_name": "GPT-5.6 Terra",
"model_id": "gpt-5.6-terra",
"date": "2026-07-23T15:24:44.541455",
"shots": 2,
"config": {
"model_id": "gpt-5.6-terra",
"display_name": "GPT-5.6 Terra",
"shots": 2,
"temperature": 0.0,
"reasoning_effort": "high",
"date": "2026-07-23",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:21:00.747970",
"file_results": [
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:26.448164",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:21:00.747970",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both apply the same core change\u2014adding a period to convert \"Fellow Travellers\" into \"Fellow Travellers.\" The original and corrected texts are identical between ground truth and prediction, capturing the intended punctuation insertion precisely."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change\u2014adding a comma after \"howling\". Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the identical core change\u2014adding a comma after \"yonder\". Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\nText Content Accuracy: The predicted change converts \"Sir\" to \"sir,\" identical to the ground truth\u2019s original_text and corrected_text. This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same core change, inserting a comma after \"pockets\" (\"pockets and\" \u2192 \"pockets, and\"). There is no mismatch in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:23.818497",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:33:02.110370",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling",
"corrected_text": "howling,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"howling over\" to \"howling, over\", which is essentially inserting a comma after the word \"howling\" (i.e., between \"howling\" and \"over\"). The prediction captures this same core change by converting \"howling\" to \"howling,\". Although it omits the following word \"over\" from the span, it still correctly represents the intended punctuation insertion at the same location."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder",
"corrected_text": "yonder,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after \"yonder\" in the phrase \"yonder to-day\" \u2192 \"yonder, to-day\". The prediction also inserts a comma after \"yonder\" (\"yonder\" \u2192 \"yonder,\"). Although the prediction includes less surrounding context (it omits \"to-day\"), it still captures the core intended punctuation change: adding a comma immediately after \"yonder\"."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"Sir\" to \"sir,\" capturing the same core capitalization correction with identical original and corrected text.\nTherefore, the prediction correctly matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"pockets and\" to \"pockets, and\", inserting a comma after \"pockets\". The core intended punctuation change is captured identically.\nTherefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day",
"corrected_text": "to-day,",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "harbour",
"corrected_text": "Harbour",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "terrace gardens",
"corrected_text": "terrace-gardens",
"observed_line_number": 33,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "So",
"corrected_text": "so",
"observed_line_number": 35,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:24.240888",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:33:41.139775",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. The core intended punctuation change is captured identically. Therefore, the predicted edit correctly matches the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change: insertion of a comma after \"howling.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the intended insertion of a comma after \"yonder\". No discrepancies in the core change."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"pockets and\" to \"pockets, and\" by inserting a comma after \"pockets.\" This captures the core intended punctuation correction with identical original and corrected text spans.\n\nTherefore, the prediction correctly matches the ground truth on both criteria."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to-day Sir",
"corrected_text": "to-day, Sir",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:21.157068",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:21:11.422958",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"No sir\" to \"No, sir,\" inserting a comma after \"No.\" The predicted edit captures the core intended change exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones?",
"corrected_text": "stones?\u201d",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "why",
"corrected_text": "\u201cwhy",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:14.981644",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:32:40.997164",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"capitalization\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the initial \"why\" in the phrase \"why did he dine\" to \"Why\" (i.e., capitalizing the first word). The prediction captures this same core change by editing \"why\" \u2192 \"Why\". Although it provides less surrounding context than the ground truth, it still reflects the essential correction intended."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "think, ",
"corrected_text": "think, \u201c",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones?",
"corrected_text": "stones?\u201d",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:26.281751",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:33:35.201524",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"No sir\") and the same correction (\"No, sir\"), inserting a comma after \"No.\" This fully captures the core intended edit with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the initial \"why\" in the phrase \"why did he dine\" to \"Why\" (i.e., capitalizing the first letter at the start of the sentence/phrase). The prediction captures this same core change by specifying \"why\" \u2192 \"Why\". Although it includes less surrounding context than the ground truth, it still reflects the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:25.289577",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:20:51.406133",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the word \"if\" in the phrase \"if he\" \u2192 \"If he.\" The prediction captures the same essential change (\"if\" \u2192 \"If\"). Although it omits the following word \"he\" from the original/corrected text span, it still reflects the core intended correction: capitalizing \"if.\" Therefore, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the verb \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam.\" The predicted edit replaces \"said\" with \"asked\" without including the following context (\"Clennam\"), but it still captures the essential intended change (said \u2192 asked). Additional surrounding context is optional, so this is considered a correct match to the ground truth\u2019s core correction."
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:23.327117",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:32:26.402774",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth replaces \"said\" with \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam.\" The predicted edit captures the same core substitution (\"said\" \u2192 \"asked\"), merely omitting the surrounding context (\"Clennam\"). Since the essential change is correctly represented, the content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:16.289194",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:33:14.480901",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches.\n\n2) Text Content Accuracy: The ground truth changes the word \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The predicted edit captures the same core substitution (\"said\" \u2192 \"asked\"), even though it omits the surrounding context \"Clennam\". Since the essential intended change is correctly identified, the content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:14.079548",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:21:15.981762",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"high road\" to \"high-road,\" capturing the identical core edit (inserting a hyphen between the words). Therefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"materials costing\" to \"materials, costing\", inserting a comma after \"materials\". The core change is identical, so the text content matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"about and\") and the same correction (\"about, and\"), inserting a comma after \"about\". This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association,",
"corrected_text": "association",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change removes the comma after \"association\" in the phrase \"association, did\" \u2192 \"association did\". The predicted edit removes the comma after \"association\" as well (\"association,\" \u2192 \"association\"). Although the prediction includes less surrounding context (it omits \"did\"), it captures the core intended change: deleting the comma after \"association\". \u0644\u0630\u0644\u0643 the essential punctuation correction matches the ground truth's intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"earnestly telling\") and the same corrected text (\"earnestly, telling\"), capturing the identical core change of inserting a comma after \"earnestly\". No discrepancies in the intended punctuation change."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change: adding a comma after \"said\" (\"said\" \u2192 \"said,\"). There is no discrepancy in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"Clennam\" (\"Clennam in\" \u0000 \"Clennam, in\"). The original_text and corrected_text are identical between ground truth and prediction, capturing the intended punctuation correction precisely."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said,",
"corrected_text": "said",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:15.406212",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:32:49.825816",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change: insertion of a comma after \"materials.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, inserting a comma after \"about\" to change \"about and\" to \"about, and.\" There is no discrepancy in original or corrected text content. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same original text (\"association, did\") and the same corrected text (\"association did\"), capturing the intended removal of the comma between \"association\" and \"did.\" Therefore, the core change matches the ground truth."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change: inserting a comma after \"earnestly\" in the phrase \"earnestly telling\" to become \"earnestly, telling\". The core punctuation change is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"said when\" to \"said, when,\" inserting a comma after \"said.\" This captures the core intended change with no discrepancy."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"said\" is corrected to \"said,\" by adding a comma. The original_text and corrected_text match the ground truth precisely. Therefore, the predicted edit correctly captures the intended correction."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"Clennam\" (\"Clennam in\" \u2192 \"Clennam, in\"). The core punctuation correction is identical. Therefore the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "highroad",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:12.790216",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:33:44.399442",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same change from \"materials costing\" to \"materials, costing,\" inserting a comma after \"materials.\" This captures the core intended correction with identical original and corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both identify the same original text (\"about and\") and the same correction (\"about, and\"), inserting a comma after \"about\". The core change is identical, with no discrepancy in the intended punctuation adjustment."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original span \"earnestly telling\" and the same correction \"earnestly, telling\", inserting a comma after \"earnestly\". The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"said when\") and the same corrected text (\"said, when\"), capturing the identical core change\u2014adding a comma after \"said.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"said\" \u2192 \"said,\", i.e., inserting a comma after the word \"said\". There is no discrepancy in the original or corrected text. Therefore, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text (\"Clennam in\") and the same corrected text (\"Clennam, in\"), capturing the identical core change\u2014adding a comma after \"Clennam.\" Therefore, the predicted edit fully matches the ground truth intent."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "highroad",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "associa-tion",
"corrected_text": "association",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:52.981049",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:20:50.336340",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Shadow,",
"corrected_text": "Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:55.180840",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:32:20.407476",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Sun and Shadow,",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:22:55.882455",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:33:14.040190",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ",",
"corrected_text": ".",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:03.543495",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:21:02.200850",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original span \"dull autumn\" and the same corrected span \"dull, autumn\", inserting a comma after \"dull\". The core punctuation change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"Saone\" and the same corrected text \"Sa\u00f4ne\" (adding the circumflex on 'o'). The core change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\nText Content Accuracy: Both ground truth and prediction change \"man slowly\" to \"man, slowly,\" inserting a comma after \"man.\" The core punctuation change is identical."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "poplar-trees",
"corrected_text": "poplar trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone it",
"corrected_text": "Saone, it",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:08.693698",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:32:32.797049",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"dull autumn\") and the same corrected text (\"dull, autumn\"), capturing the identical core change: insertion of a comma after \"dull\". Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit have type \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"man slowly\" to \"man, slowly\", correctly capturing the intended insertion of a comma after \"man\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:12.437633",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:33:43.767992",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"dull autumn\") and the same correction (\"dull, autumn\"), inserting a comma after \"dull\". This captures the core intended change with no discrepancies."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same transformation from \"man slowly\" to \"man, slowly\", correctly capturing the intended insertion of a comma after \"man\". Therefore, the predicted edit matches the ground truth in both type and core textual change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"Chalons was\") and the same corrected text (\"Chalons, was\"), capturing the identical core change\u2014adding a comma after \"Chalons.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:34.827419",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:21:28.746751",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"heart so\") and the same corrected text (\"heart, so\"), capturing the identical core change\u2014adding a comma after \"heart.\" Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "at length said",
"corrected_text": "at length, said",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma between \"length\" and \"said\" (\"length said\" \u2192 \"length, said\"). The prediction performs the same core punctuation insertion, but includes additional surrounding context (\"at length said\" \u2192 \"at length, said\"). This still captures the essential intended edit: adding a comma after \"length\" before \"said.\" \u05dc\u05db\u05df the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change from present to past tense. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "she thanked",
"corrected_text": "she indeed thanked",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:28.846378",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:32:50.516087",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart",
"corrected_text": "heart,",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"heart\" in the sequence \"heart so\" \u2192 \"heart, so.\" The predicted edit \"heart\" \u2192 \"heart,\" captures this same core punctuation insertion (adding a comma immediately after \"heart\"). Although the prediction omits the following word \"so,\" it still represents the essential intended correction: placing a comma after \"heart.\""
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement,\" so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"I hope so\") and the same corrected text (\"I hoped so\"), capturing the identical core change (present tense to past tense). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "thanked",
"corrected_text": "indeed thanked",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling;",
"corrected_text": "smiling,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:28.534345",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:33:35.422855",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"heart\" to change \"heart so\" to \"heart, so\"\u2014with identical original and corrected text. Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the same original text \"I hope so\" with the same corrected text \"I hoped so\". The core change (tense shift from hope \u2192 hoped) is captured identically.\nTherefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "she thanked",
"corrected_text": "she indeed thanked",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling;",
"corrected_text": "smiling,",
"observed_line_number": 31,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:49.308479",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:21:46.961999",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n\nText Content Accuracy: Both ground truth and prediction change \"hers and\" to \"hers, and,\" inserting a comma after \"hers.\" The core punctuation change is identical with no discrepancy in original or corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The core textual correction is identical. Therefore the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change: inserting a comma after \"moment\" (\"moment with\" \u2192 \"moment, with\"). The original and corrected text are identical between ground truth and prediction, capturing the core intended punctuation change."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"beautiful that\") and the same corrected text (\"beautiful, that\"), capturing the identical core change\u2014adding a comma after \"beautiful\". Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:52.661067",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:33:03.976909",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change: inserting a comma after \"hers\" in the phrase \"hers and\", resulting in \"hers, and\". The original_text and corrected_text match the ground truth\u2019s core change exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core punctuation change: inserting a comma after \"hers\" (\"hers matters\" \u2192 \"hers, matters\"). The original and corrected texts are identical between ground truth and prediction, capturing the intended edit precisely."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"moment with\" to \"moment, with\", inserting a comma after \"moment\". The core intended punctuation correction is captured identically."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original span (\"beautiful that\") and the same correction (\"beautiful, that\"), inserting a comma after \"beautiful\". The core intended punctuation change is captured precisely."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly. 2) Text Content Accuracy: Both ground truth and prediction change \"they\" to \"They,\" capturing the same core capitalization correction with identical original and corrected text. Therefore, the prediction matches the ground truth\u2019s intention."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:23:53.185595",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:34:00.125774",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original_text (\"hers and\") and the same corrected_text (\"hers, and\"), capturing the identical core change of inserting a comma after \"hers\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"hers matters\") and the same corrected text (\"hers, matters\"), capturing the identical core change\u2014adding a comma after \"hers\". Therefore, the predicted edit correctly reflects the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change\u2014adding a comma after \"moment\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth type.\n2) Text Content Accuracy: Both ground truth and prediction change \"beautiful that\" to \"beautiful, that\" by inserting a comma after \"beautiful.\" This captures the exact core punctuation correction with identical original and corrected text.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, converting \"they\" to \"They\". There is no discrepancy in the original or corrected text. Therefore, the predicted edit accurately captures the ground truth intention."
}
],
"file_id": "010"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:07.853057",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:21:37.216703",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the same original text \"shore and\" and the same corrected text \"shore, and,\" capturing the identical core change (inserting a comma after \"shore\"). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 32,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly.\n\nText Content Accuracy: Both specify replacing the original text \"that\" with the corrected text \"than\". This captures the core intended change exactly. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you\u201d",
"corrected_text": "Thank you,\u201d",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:03.734231",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:33:26.529880",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original segment (\"shore and\") and the same correction (\"shore, and\"), capturing the core intended change of inserting a comma after \"shore\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 32,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both edits replace the original text \"that\" with the corrected text \"than\", capturing the same core change. Line numbers are ignored as instructed, so the mismatch there does not affect correctness."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you\u201d",
"corrected_text": "Thank you,\u201d",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 27,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:00.637285",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:34:10.224488",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same original text (\"shore and\") and the same corrected text (\"shore, and\"), capturing the identical core change of inserting a comma after \"shore\". Therefore, the predicted edit matches the ground truth\u2019s intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both replace the original text \"that\" with the corrected text \"than\". The core intended change is captured with no discrepancies. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that in",
"corrected_text": "that, in",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy whatever",
"corrected_text": "fancy, whatever",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you\u201d",
"corrected_text": "Thank you,\u201d",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:16.066025",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:22:06.184292",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\". The original and corrected texts are identical between ground truth and prediction, so the intended punctuation correction is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth.\n\nText Content Accuracy: The predicted change is \"city\" \u2192 \"City,\" which is identical to the ground truth and captures the core intended capitalization correction without altering any other content.\n\nTherefore, the prediction correctly matches the ground truth edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change: replacing \"them,\" with \"them;\". There is no discrepancy in original or corrected text. Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"it,\") and the same corrected text (\"it;\"), capturing the identical core change from comma to semicolon. Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text \"stand-still,\" and the same corrected text \"stand-still;\". The core change\u2014replacing the comma with a semicolon\u2014is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: Both specify the same transformation: \"existence,\" \u2192 \"existence;\". The core intended punctuation change (comma to semicolon) is captured precisely with identical original and corrected text.\n\nTherefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch Goes on Dreaming",
"corrected_text": "Mrs. Flintwinch Goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,",
"corrected_text": "view;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:44.541455",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:33:47.216953",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a period to \"Chapter XXIX\" to make it \"Chapter XXIX.\" The original_text and corrected_text align perfectly with the ground truth intention.\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "MRS. FLINTWINCH GOES ON DREAMING",
"corrected_text": "MRS. FLINTWINCH GOES ON DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth edit adds a period after the word \"DREAMING\" (\"DREAMING\" \u2192 \"DREAMING.\"). The predicted edit adds a period at the end of a longer phrase whose final word is \"DREAMING\" (\"... DREAMING\" \u2192 \"... DREAMING.\"). This includes extra surrounding context but preserves the same core punctuation change\u2014adding a period immediately after \"DREAMING\". \u0644\u0630\u0644\u0643 the prediction captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\nText Content Accuracy: Both ground truth and prediction change \"city\" to \"City,\" capturing the identical core correction (capitalizing the first letter). Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"replacement\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify replacing \"of\" with \"to\". The core textual change is identical and fully captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a comma after \"existence\" (\"existence\" \u2192 \"existence,\"). The corrected text matches the ground truth\u2019s intention with no discrepancy."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the identical core change: adding a comma after \"motionless\" (\"motionless\" \u2192 \"motionless,\"). The predicted edit fully captures the intended correction with no discrepancy."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"view\" (\"view\" \u2192 \"view,\"). The predicted edit fully captures the ground truth\u2019s intended punctuation correction."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to make the punctuation after \"existence\" a semicolon (i.e., \"existence,\" \u2192 \"existence;\"). The prediction also results in \"existence;\" as the corrected text. Although the predicted original_text omits the comma (\"existence\" instead of \"existence,\"), it still captures the core intended change: the word should end with a semicolon rather than whatever punctuation was there before. Therefore, the essential correction (ending punctuation becomes a semicolon) is correctly represented."
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them",
"corrected_text": "them,",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "lord",
"corrected_text": "lord,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:35.038425",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:34:29.167579",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both apply the same core change, adding a period to transform \"Chapter XXIX\" into \"Chapter XXIX.\". The original and corrected texts are identical between ground truth and prediction, so the intended edit is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "MRS. FLINTWINCH GOES ON DREAMING",
"corrected_text": "MRS. FLINTWINCH GOES ON DREAMING.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period after the word \"DREAMING\" (\"DREAMING\" \u2192 \"DREAMING.\"). The predicted edit adds a period after \"DREAMING\" within the longer phrase (\"MRS. FLINTWINCH GOES ON DREAMING\" \u2192 \"MRS. FLINTWINCH GOES ON DREAMING.\"). This includes additional surrounding context but captures the same core punctuation change at the same target word. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"capitalization\" and the predicted edit type is also \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both ground truth and prediction change \"city\" to \"City\", capturing the same core capitalization correction with identical original and corrected text. Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 9,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core replacement: \"them,\" \u2192 \"them;\". This captures the identical intended punctuation correction. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it;",
"corrected_text": "it,",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still;",
"corrected_text": "stand-still,",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege lord",
"corrected_text": "liege-lord",
"observed_line_number": 24,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "always",
"corrected_text": "always to",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view,",
"corrected_text": "view",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence;",
"corrected_text": "existence",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:32.276038",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:22:11.183386",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"now\" (\"now I\" \u2192 \"now, I\"). The original and corrected text strings are identical between ground truth and prediction, so the intended correction is fully captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches exactly. Text Content Accuracy: Both specify replacing \"authorize\" with \"authorise\", capturing the identical core spelling change. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam; \u201cnot",
"corrected_text": "Clennam, \u201cnot",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:34.865214",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:34:08.487705",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edit specify the type as \"italicize\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same target text \"him\" with no change in wording other than applying italics. The prediction captures the same core intention: italicize the word \"him\". Therefore, the predicted edit matches the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"now I\" to \"now, I\" (inserting a comma after \"now\"). The core intended punctuation correction is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: replacing \"authorize\" with \"authorise\". There is no discrepancy in original or corrected text. Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "eyes by",
"corrected_text": "eyes, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "white but",
"corrected_text": "white, but",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a little",
"corrected_text": "a little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now to",
"corrected_text": "now, to",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you now",
"corrected_text": "you, now",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:33.156121",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:34:27.022747",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"now I\" to \"now, I,\" inserting a comma after \"now.\" This captures the core intended change precisely with no discrepancy."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both identify the same substitution of \"authorize\" with \"authorise\". The core change is identical, with no discrepancy in original or corrected text.\nTherefore, the predicted edit fully matches the ground truth intent."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "a little",
"corrected_text": "a little",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him",
"corrected_text": "benevolence bubbling out of him",
"observed_line_number": 30,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:42.337757",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T11:22:20.109628",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed.",
"corrected_text": "bed;",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "bed-room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "For",
"corrected_text": "for",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": ",that",
"corrected_text": ", that",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me,",
"corrected_text": "me;",
"observed_line_number": 22,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:36.889405",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T12:34:04.789859",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly,",
"corrected_text": "quickly",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me,",
"corrected_text": "me\u2014",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "For Mr. Pancks",
"corrected_text": "For, Mr. Pancks",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "gpt-5.6-terra",
"date": "2026-07-23T15:24:39.968584",
"annotator": "GPT-5.6 Terra",
"annotation_date": "2026-07-23T13:34:31.201756",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For Mr.",
"corrected_text": "For, Mr.",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The predicted edit similarly inserts a comma after \"For\" in the shorter span \"For Mr.\" \u2192 \"For, Mr.\". Although the prediction includes less surrounding context (it omits \"Pancks\"), it captures the core intended punctuation change (adding the comma after \"For\"). Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly that",
"corrected_text": "quickly, that",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation from \"quickly that\" to \"quickly, that\" (inserting a comma after \"quickly\"). The core change is identical, with no discrepancies in original or corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bedroom",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "me not",
"corrected_text": "me, not",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2026-07-09",
"release_source": "https://community.openai.com/t/introducing-gpt-5-6-series-sol-terra-and-luna-coming-july-9-10am-pt/1384931",
"release_notes": "Mid tier of the GPT-5.6 series; OpenAI positions Terra as matching GPT-5.5."
},
{
"model_name": "GPT-6 Astra (OR)",
"model_id": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:43.787920",
"shots": 2,
"config": {
"model_id": "or_openai_gpt_6_astra",
"display_name": "GPT-6 Astra (OR)",
"shots": 2,
"temperature": 0.0,
"reasoning_effort": "high",
"date": "2026-09-06",
"notes": "Benchmark run with 2-shot learning at high reasoning effort"
},
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:19.772798",
"file_results": [
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:02.683351",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:19.772798",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth intends to add a period at the end of the heading/phrase \"Fellow Travellers\" (i.e., \"Fellow Travellers\" \u2192 \"Fellow Travellers.\"). The prediction applies the same core change\u2014adding a period to the final word \"Travellers\" (\"Travellers\" \u2192 \"Travellers.\"). Although the prediction omits the preceding word \"Fellow\" in its span, it still captures the essential intended correction: inserting a period after \"Travellers\" at the end of the phrase."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling",
"corrected_text": "howling,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after the word \"howling\" in the phrase \"howling over\" (i.e., \"howling over\" \u2192 \"howling, over\"). The predicted edit captures the same core punctuation action by changing \"howling\" \u2192 \"howling,\". Although it omits the following word \"over\" from the span, it still represents inserting the comma at the correct location (after \"howling\"), which is the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder",
"corrected_text": "yonder,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"yonder\" in the phrase \"yonder to-day\" \u2192 \"yonder, to-day.\" The predicted edit \"yonder\" \u2192 \"yonder,\" captures the same essential punctuation insertion (a comma immediately following \"yonder\"). Although it omits the trailing context (\"to-day\"), it still reflects the core intended change."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"Sir\" to \"sir\". The core intended change (lowercasing the initial letter) is captured identically. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets",
"corrected_text": "pockets,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"pockets\" in the phrase \"pockets and\" \u2192 \"pockets, and.\" The prediction captures the same core punctuation change by editing \"pockets\" \u2192 \"pockets,\". Although it omits the following word \"and\" as context, it still correctly represents the essential intended edit: adding a comma immediately after \"pockets.\""
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:00.507726",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:20:46.410226",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period at the end of the phrase \"Fellow Travellers\" (i.e., after \"Travellers\"). The predicted edit adds a period after \"Travellers\" as well, just with less surrounding context (it omits \"Fellow\"). Since the essential correction\u2014adding the terminal period after \"Travellers\"\u2014is captured, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling",
"corrected_text": "howling,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"howling\" in the phrase \"howling over\" to make it \"howling, over\". The prediction edits \"howling\" to \"howling,\", which captures the same essential punctuation change (adding the comma immediately after \"howling\"). The omission of the following word \"over\" is acceptable because the core correction\u2014the comma insertion after \"howling\"\u2014is correctly represented."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder",
"corrected_text": "yonder,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth edit inserts a comma after the word \"yonder\" in the phrase \"yonder to-day\" \u2192 \"yonder, to-day\". The predicted edit \"yonder\" \u2192 \"yonder,\" captures this core intended change (adding a comma immediately after \"yonder\"). Although the prediction omits the following word \"to-day\" from its span, it still correctly represents the essential punctuation correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change from \"Sir\" to \"sir\" (lowercasing the initial letter). There is no discrepancy in original or corrected text. Therefore, the prediction captures the ground truth edit precisely."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets",
"corrected_text": "pockets,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after the word \"pockets\" (changing \"pockets and\" to \"pockets, and\"). The prediction also inserts a comma after \"pockets\" (\"pockets\" \u2192 \"pockets,\"). Although it includes less surrounding context (omitting the following \"and\"), it captures the same core punctuation change the ground truth specifies."
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:06.360904",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:22:46.447264",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Travellers",
"corrected_text": "Travellers.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period at the end of the title/phrase \"Fellow Travellers\" \u2192 \"Fellow Travellers.\". The prediction applies the same essential punctuation change to the final word \"Travellers\" \u2192 \"Travellers.\". Although it omits the preceding word \"Fellow\", it still captures the core intended edit: insert a period at the end of the heading/phrase. Additional surrounding context is not required, so this is considered correct."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling",
"corrected_text": "howling,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"howling\" in the phrase \"howling over\" (i.e., \"howling over\" \u2192 \"howling, over\"). The predicted edit captures this core punctuation action by changing \"howling\" \u2192 \"howling,\". Although it includes less surrounding context (it does not explicitly include the following word \"over\"), it still represents the same essential correction: adding a comma immediately after \"howling.\" Therefore, the prediction matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "yonder",
"corrected_text": "yonder,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"yonder\" in the phrase \"yonder to-day\" \u2192 \"yonder, to-day\". The predicted edit \"yonder\" \u2192 \"yonder,\" captures the same core punctuation change (adding the comma immediately after \"yonder\"), even though it includes less surrounding context. Since the essential intended modification is correctly represented, the content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core transformation from \"Sir\" to \"sir\". There is no mismatch in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "pockets",
"corrected_text": "pockets,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\nText Content Accuracy: The ground truth inserts a comma after \"pockets\" in the phrase \"pockets and\" \u2192 \"pockets, and.\" The prediction also captures this core change by converting \"pockets\" \u2192 \"pockets,\" (adding the comma immediately after the word). Although the prediction omits the following word \"and\" from the context, it still represents the same essential punctuation insertion at the correct location. Therefore, the predicted edit matches the ground truth\u2019s intention."
}
],
"file_id": "003"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:24:55.699056",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:30.098494",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No",
"corrected_text": "No,",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"No\" in the phrase \"No sir\" \u2192 \"No, sir.\" The predicted edit \"No\" \u2192 \"No,\" captures the same core punctuation change (adding a comma immediately after \"No\"). Although it omits the following word \"sir\" as context, it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase (\"why did he dine\" \u2192 \"Why did he dine\"). The prediction captures the same core change by converting \"why\" to \"Why.\" Although it includes less surrounding context, it correctly represents the essential capitalization correction intended in the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation point to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction captures the same core change (\"!\" \u2192 \"?\") on the key word segment \"stones!\" \u2192 \"stones?\". Although it includes less surrounding context (omits \"cold\"), it still reflects the essential intended correction to the punctuation at the end of the phrase."
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:24:56.420552",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:20:50.736983",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No",
"corrected_text": "No,",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth correction changes \"No sir\" to \"No, sir,\" which is specifically the insertion of a comma immediately after \"No.\" The predicted edit changes \"No\" to \"No,\", capturing the same essential punctuation insertion at the correct location. Although the prediction omits the following word \"sir\" (less context), it still represents the core intended change: adding the comma after \"No\"."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"capitalization,\" which exactly matches the ground truth type.\n\n2) Text Content Accuracy: The ground truth capitalizes the first word of the phrase \"why did he dine\" to \"Why did he dine.\" The prediction captures the same core change by converting \"why\" to \"Why.\" Although it provides less surrounding context, it correctly represents the essential intended correction (capitalizing the initial 'w' in 'why')."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the ending punctuation from an exclamation mark to a question mark (\"cold stones!\" \u2192 \"cold stones?\"). The prediction applies the same core change to the same word sequence at the point of punctuation (\"stones!\" \u2192 \"stones?\") but with less surrounding context. Since the essential intended edit is the punctuation swap ! \u2192 ? after \u201cstones\u201d, the prediction correctly captures the ground truth\u2019s intention."
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:00.329981",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:23:03.650109",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "No",
"corrected_text": "No,",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"No sir\" to \"No, sir\", which is the insertion of a comma immediately after the word \"No\". The prediction changes \"No\" to \"No,\", which captures the same core punctuation change (adding a comma after \"No\"). Although the prediction includes less surrounding context (it omits \"sir\"), it still correctly represents the essential intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 28,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: The ground truth change is to capitalize the initial word in the phrase \"why did he dine\" \u2192 \"Why did he dine\". The prediction captures the same core change by converting \"why\" \u2192 \"Why\". Although it provides less surrounding context than the ground truth, it correctly represents the essential intended correction (capitalizing \"why\")."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones!",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation point to a question mark in the phrase \"cold stones!\" \u2192 \"cold stones?\". The prediction captures the same core change at the same word boundary: \"stones!\" \u2192 \"stones?\". Although it omits the preceding word \"cold\", it still correctly represents the essential intended edit (replacing ! with ? after \"stones\"). Additional or reduced context is acceptable as long as the core change is preserved.\n\nTherefore, the predicted edit matches the ground truth\u2019s intention."
}
],
"file_id": "004"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:24:53.719810",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:25.969520",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to capitalize the initial word \"if\" in the phrase \"if he\" \u2192 \"If he\". The prediction captures the same core change by capitalizing \"if\" \u2192 \"If\"; it simply provides a narrower span of text (omitting the following word \"he\"). Since the essential correction\u2014capitalizing \"if\"\u2014is correctly represented, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to change the verb in the phrase \"said Clennam\" to \"asked Clennam.\" The prediction captures the core change by replacing \"said\" with \"asked.\" Although it omits the surrounding context word \"Clennam\" in the original/corrected spans, it still represents the essential intended correction (said \u2192 asked). Therefore, the text content matches the ground truth\u2019s intention."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:24:56.631470",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:20:40.222792",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is capitalizing the initial word in the phrase \"if he\" \u2192 \"If he\". The prediction captures the core change by capitalizing \"if\" \u2192 \"If\" (with less surrounding context). Since the essential intended correction is the capitalization of \"if\", the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the type matches exactly.\n2. Text Content Accuracy: The ground truth changes \"said Clennam\" to \"asked Clennam\". The prediction changes \"said\" to \"asked\" without including the following word \"Clennam\", but it still captures the core intended change (replacing \"said\" with \"asked\"). Additional surrounding context is optional, so this is acceptable."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:01.677414",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:22:50.218460",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the word \"if\" at the start of the phrase \"if he\" (\"if he\" \u2192 \"If he\"). The prediction applies the same capitalization change to the core word itself (\"if\" \u2192 \"If\"). Although it includes less surrounding context (omits \"he\"), it still captures the essential intended correction: capitalizing \"if\". Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"said\" to \"asked\" in the phrase \"said Clennam\" \u2192 \"asked Clennam\". The prediction captures the same core substitution (\"said\" \u2192 \"asked\"). Although it omits the surrounding context word \"Clennam\" in the original/corrected_text fields, the essential intended change is correctly represented and does not alter the meaning of the edit."
}
],
"file_id": "005"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:16.205529",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:53.055908",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly. Text Content Accuracy: Both specify the same core change, converting \"high road\" to the hyphenated form \"high-road.\" There is no mismatch in the original or corrected text, so the prediction captures the intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"materials\" in the phrase \"materials costing\" \u2192 \"materials, costing\". The predicted edit captures this same core change by converting \"materials\" \u2192 \"materials,\". Although it does not include the following word \"costing\" in the span, it still correctly represents the essential punctuation insertion intended by the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"about\" in the phrase \"about and\" \u2192 \"about, and\". The prediction captures the same core punctuation change by editing \"about\" \u2192 \"about,\". Although it omits the following word \"and\" from its span, it still correctly represents the essential intended correction (adding a comma immediately after \"about\")."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association,",
"corrected_text": "association",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2. Text Content Accuracy: The ground truth intends to remove the comma after \"association\" (changing \"association, did\" to \"association did\"). The prediction removes the comma after \"association\" as well (\"association,\" \u2192 \"association\"). Although the prediction includes less surrounding context (it omits \"did\"), it captures the core intended change: deletion of the comma following \"association.\""
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly",
"corrected_text": "earnestly,",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"earnestly\" in the phrase \"earnestly telling\" \u2192 \"earnestly, telling.\" The prediction captures the same core change by converting \"earnestly\" \u2192 \"earnestly,\". Although it includes less surrounding context (it omits the following word \"telling\"), it still correctly represents the essential punctuation edit: adding a comma immediately after \"earnestly.\""
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"said\" in the phrase \"said when\" \u2192 \"said, when\". The predicted edit captures the same core punctuation change by converting \"said\" \u2192 \"said,\", which is precisely the comma insertion required before \"when\". Although the prediction provides less surrounding context (it omits \"when\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after \"Clennam\" in the phrase \"Clennam in\" \u2192 \"Clennam, in\". The prediction specifies \"Clennam\" \u2192 \"Clennam,\", which captures the same essential punctuation insertion (adding a comma immediately after \"Clennam\"). Although the prediction omits the following word \"in\", it still represents the core intended change."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said:",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:24.180703",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:21:09.984674",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high\u00b7road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to convert \u201chigh road\u201d into the hyphenated form \u201chigh-road.\u201d The prediction also yields the same corrected text \u201chigh-road\u201d and targets the same compound (\u201chigh\u201d + \u201croad\u201d). Although the predicted original uses a middle dot (\"high\u00b7road\") instead of a space (\"high road\"), the essential editorial intention\u2014hyphenating the compound\u2014is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"materials\" in the phrase \"materials costing\" to make it \"materials, costing.\" The prediction captures the same core punctuation change by changing \"materials\" to \"materials,\" (it omits the following word \"costing,\" but that context is not required as long as the essential edit\u2014adding the comma after \"materials\"\u2014is represented)."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"about\" in the phrase \"about and\" (i.e., \"about and\" \u2192 \"about, and\"). The prediction captures the same core punctuation change by converting \"about\" \u2192 \"about,\". Although it omits the following word \"and\" from the edit span, it still reflects the essential intended correction: adding a comma after \"about\" before \"and\"."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association,",
"corrected_text": "association",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth removes the comma after \"association\" in the sequence \"association, did\" \u2192 \"association did\". The predicted edit \"association,\" \u2192 \"association\" captures the same core change (deleting the comma after \"association\"). Although the prediction includes less surrounding context (it omits \"did\"), it still correctly represents the essential punctuation correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly",
"corrected_text": "earnestly,",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"earnestly\" in the phrase \"earnestly telling\" \u2192 \"earnestly, telling\". The predicted edit \"earnestly\" \u2192 \"earnestly,\" captures the same core punctuation action (adding a comma immediately after \"earnestly\"). Although the prediction includes less surrounding context (it omits \"telling\"), it still reflects the essential intended correction: inserting the comma at that position."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"said\" in the phrase \"said when\" \u2192 \"said, when\". The predicted edit \"said\" \u2192 \"said,\" captures the same core punctuation insertion (adding a comma immediately after \"said\"). Although it includes less surrounding context (it does not mention the following word \"when\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"Clennam\" in the phrase \"Clennam in\" \u2192 \"Clennam, in.\" The predicted edit \"Clennam\" \u2192 \"Clennam,\" captures the same essential punctuation insertion (a comma immediately after \"Clennam\"). Although it omits the following word \"in\" from the context, it still reflects the core intended change."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said:",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:21.370259",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:23:16.948931",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation,\" so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"high road\" to \"high-road\" (inserting a hyphen). The core intended correction is captured with identical original and corrected text. Therefore, the prediction matches the ground truth intent."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after the word \"materials\" (changing \"materials costing\" to \"materials, costing\"). The predicted edit captures this same core change by changing \"materials\" to \"materials,\". Although the prediction does not include the following word \"costing\" in its original/corrected text span, it still correctly represents the essential punctuation insertion at the correct location (immediately after \"materials\")."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about",
"corrected_text": "about,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"about\" in the phrase \"about and\" \u2192 \"about, and\". The prediction captures the same core punctuation change by converting \"about\" \u2192 \"about,\". Although it omits the following word \"and\" in its span, it still correctly reflects the essential intended edit: adding a comma after \"about\" before \"and\"."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association,",
"corrected_text": "association",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change removes the comma after \u201cassociation\u201d so that \u201cassociation, did\u201d becomes \u201cassociation did.\u201d The predicted edit removes the comma after \u201cassociation\u201d (\u201cassociation,\u201d \u2192 \u201cassociation\u201d). Although it includes less surrounding context (it omits \u201cdid\u201d), it captures the core intended correction: deleting the comma after \u201cassociation.\u201d Therefore, the text-content intention matches."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly",
"corrected_text": "earnestly,",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"earnestly\" in the phrase \"earnestly telling\" \u2192 \"earnestly, telling\". The prediction captures the same essential change by editing \"earnestly\" \u2192 \"earnestly,\". Although it includes less surrounding context (it omits the following word \"telling\"), it still correctly represents the intended punctuation insertion at the same location."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after \"said\" in the phrase \"said when\" \u2192 \"said, when\". The predicted edit \"said\" \u2192 \"said,\" captures the same core punctuation change (adding a comma immediately after \"said\"). Even though it omits the following word \"when\" from the span, it still reflects the essential intended correction."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam",
"corrected_text": "Clennam,",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after \"Clennam\" in the phrase \"Clennam in\" \u2192 \"Clennam, in\". The prediction captures the same core punctuation change by editing \"Clennam\" \u2192 \"Clennam,\". Although it omits the following word \"in\" as context, it still applies the comma insertion to the correct token and reflects the essential intended correction."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said:",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:25.219039",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:17.205581",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Shadow",
"corrected_text": "Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period to the end of the phrase \"Sun and Shadow\" (i.e., the period goes after the final word \"Shadow\"). The prediction captures the same core punctuation change by showing \"Shadow\" \u2192 \"Shadow.\", which places the period in the correct location. Although the prediction includes less surrounding context (it omits \"Sun and\"), it still reflects the essential intended edit: adding a terminal period after \"Shadow\"."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "1",
"corrected_text": "1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:25.302190",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:20:38.026413",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Shadow",
"corrected_text": "Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth adds a period to the end of the title \"Sun and Shadow\" (i.e., after \"Shadow\"). The prediction adds a period after \"Shadow\" as well, but includes less surrounding context (only the final word). Since the core intended change is adding a period at the end (after \"Shadow\"), the prediction captures the essential correction despite the shorter span of original/corrected text."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "I",
"corrected_text": "I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:26.821001",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:22:52.541745",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Shadow",
"corrected_text": "Shadow.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to add a period to the end of the title \"Sun and Shadow\" (i.e., after the final word \"Shadow\"). The predicted edit adds a period to \"Shadow\" \u2192 \"Shadow.\", which places the period in the same intended location (the end of the title). Although the prediction includes less surrounding context (omitting \"Sun and\"), it still captures the core change: adding terminal punctuation after the last word."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "1",
"corrected_text": "1.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
}
],
"file_id": "007"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:46.230788",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:28.590117",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Loose",
"corrected_text": "Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period at the end of the phrase \"Let Loose\" (\"Let Loose\" \u2192 \"Let Loose.\"). The predicted edit adds a period after the final word \"Loose\" (\"Loose\" \u2192 \"Loose.\"), which captures the same core intention: inserting a terminal period at the end of the relevant text. Although the prediction includes less surrounding context (omits \"Let\"), the essential punctuation change is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull",
"corrected_text": "dull,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"dull\" in the phrase \"dull autumn\" (i.e., \"dull autumn\" \u2192 \"dull, autumn\"). The prediction captures the same core punctuation change by editing \"dull\" \u2192 \"dull,\". Although it omits the following word \"autumn\" from the context, it still correctly represents the essential intended edit: adding a comma immediately after \"dull\"."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The original and corrected texts are identical between ground truth and prediction, so the core change is captured correctly."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'), which is the core intended correction. No mismatch in original or corrected text content."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"man\" in the phrase \"man slowly\" (\"man slowly\" \u2192 \"man, slowly\"). The predicted edit (\"man\" \u2192 \"man,\") captures the same essential punctuation insertion at the same location. Although it includes less surrounding context (it omits \"slowly\"), it still represents the core intended correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"Chalons\" in the phrase \"Chalons was\" \u2192 \"Chalons, was.\" The prediction also captures the essential change by converting \"Chalons\" \u2192 \"Chalons,\" (adding the comma immediately after \"Chalons\"). Although the prediction includes less surrounding context (it omits \"was\"), it still reflects the same core punctuation edit intended in the ground truth."
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XI",
"corrected_text": "XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "trees",
"corrected_text": "trees,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:46.796212",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:20:50.026289",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Loose",
"corrected_text": "Loose.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to add a period at the end of the phrase \"Let Loose\" (\"Let Loose\" \u2192 \"Let Loose.\"). The prediction captures the same core change\u2014adding a final period\u2014by applying it to the last word (\"Loose\" \u2192 \"Loose.\"). Although it includes less surrounding context, it correctly represents the essential punctuation insertion at the end of the relevant text."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull",
"corrected_text": "dull,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"dull\" in the phrase \"dull autumn\" (becoming \"dull, autumn\"). The predicted edit \"dull\" \u2192 \"dull,\" captures this same core punctuation change (adding the comma immediately after \"dull\"), even though it omits the following word \"autumn\". This is acceptable because it preserves the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. 2) Text Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the 'o'), capturing the identical core correction. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both indicate the same core change, replacing \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the o). There is no discrepancy in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is the insertion of a comma immediately after the word \"man\" (\"man slowly\" \u2192 \"man, slowly\"). The prediction also inserts a comma after \"man\" (\"man\" \u2192 \"man,\"). Although the prediction includes less surrounding context (it omits the following word \"slowly\"), it captures the essential intended punctuation change in the correct location. Therefore, the core edit is correctly represented."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"Chalons\" (\"Chalons was\" \u2192 \"Chalons, was\"). The predicted edit also inserts a comma after \"Chalons\" (\"Chalons\" \u2192 \"Chalons,\"). Although the prediction includes less surrounding context (it omits the following word \"was\"), it captures the core intended punctuation change\u2014adding the comma immediately after \"Chalons.\""
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XI",
"corrected_text": "XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "trees",
"corrected_text": "trees,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:47.084939",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:23:04.653429",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "dull",
"corrected_text": "dull,",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth intends to insert a comma after the word \"dull\" (changing \"dull autumn\" to \"dull, autumn\"). The prediction captures the same core change by changing \"dull\" to \"dull,\". Although it omits the following word \"autumn\" from the span, it still represents the essential punctuation insertion at the correct location (after \"dull\"). Therefore, the predicted edit matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth and prediction both specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both edits replace the original text \"Saone\" with the corrected text \"Sa\u00f4ne\" (adding the circumflex over the 'o'), matching the core intended change precisely. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 7,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"replacement\" and the predicted edit type is also \"replacement\", so the types match exactly.\n\nText Content Accuracy: Both edits replace \"Saone\" with \"Sa\u00f4ne\" (adding the circumflex over the o). The original_text and corrected_text match the ground truth\u2019s intended core change exactly. Therefore, the prediction captures the essential correction."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "man",
"corrected_text": "man,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"man\" in the phrase \"man slowly\" \u2192 \"man, slowly\". The prediction captures the same core change by editing \"man\" \u2192 \"man,\". Although it omits the following word \"slowly\" from the edit span, it still correctly represents the essential punctuation insertion at the correct location."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons",
"corrected_text": "Chalons,",
"observed_line_number": 9,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"Chalons\" in the phrase \"Chalons was\" \u2192 \"Chalons, was.\" The predicted edit \"Chalons\" \u2192 \"Chalons,\" captures the same core punctuation change (adding a comma immediately after \"Chalons\"). Although it omits the following word \"was\" as context, it still reflects the essential intended correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XI",
"corrected_text": "XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "trees",
"corrected_text": "trees,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Loose",
"corrected_text": "Loose.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:55.792838",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:34.099869",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart",
"corrected_text": "heart,",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2. Text Content Accuracy: The ground truth intends to insert a comma after the word \"heart\" in the phrase \"heart so,\" producing \"heart, so.\" The predicted edit changes \"heart\" to \"heart,\", which captures the same essential punctuation insertion (a comma immediately after \"heart\"). Although the prediction omits the following word \"so\" from its context, it still represents the core intended change at the correct location."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous,",
"corrected_text": "generous",
"observed_line_number": 22,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is the removal of the comma after \"generous\" (turning \"generous, Mr.\" into \"generous Mr.\"). The predicted edit also removes the comma after \"generous\" (\"generous,\" \u2192 \"generous\"). Although the prediction includes less surrounding context (it omits \"Mr.\"), it captures the core intended punctuation change: deleting the comma after \"generous.\""
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "as",
"corrected_text": "as indeed",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes \"as she did\" to \"as indeed she did,\" which is essentially the insertion of the word \"indeed\" immediately after \"as.\" The predicted edit changes \"as\" to \"as indeed,\" which captures that same core insertion, even though it provides less surrounding context (it does not include \"she did\"). Since the essential intended change\u2014adding \"indeed\" after \"as\"\u2014is correctly represented, the prediction matches the ground truth\u2019s intent."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length",
"corrected_text": "length,",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after the word \"length\" in the phrase \"length said\" \u2192 \"length, said\". The predicted edit \"length\" \u2192 \"length,\" captures the same core intention: adding a comma immediately after \"length\". Although it omits the following word \"said\" from the original/corrected text span, it still represents the essential punctuation insertion at the correct location.\n\nBecause both the type and the core textual change align with the ground truth\u2019s intention, the prediction is correct."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes the phrase \"I hope so\" to \"I hoped so\", i.e., the essential correction is changing the verb form \"hope\" \u2192 \"hoped\". The predicted edit replaces \"hope\" with \"hoped\", which captures the core intended change even though it provides less surrounding context (omits \"I\" and \"so\"). This still matches the ground truth\u2019s intention."
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:53.335446",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:21:05.837538",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart",
"corrected_text": "heart,",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are of type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after the word \"heart\" in the phrase \"heart so\" \u2192 \"heart, so\". The predicted edit \"heart\" \u2192 \"heart,\" captures the same core change (adding a comma immediately after \"heart\"). The prediction omits the following word \"so\", but that is just reduced context and does not change the essential punctuation correction.\n\nTherefore, the predicted edit correctly captures the intention of the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous,",
"corrected_text": "generous",
"observed_line_number": 22,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is removing the comma after \"generous\" in the phrase \"generous, Mr.\" \u2192 \"generous Mr.\". The prediction captures this same core punctuation change by converting \"generous,\" \u2192 \"generous\". Although the prediction omits the following context (\"Mr.\"), it still represents the essential intended correction: deleting the comma after \"generous\"."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "as",
"corrected_text": "as indeed",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches.\n\n2. Text Content Accuracy: The ground truth change is to insert the word \"indeed\" into the phrase \"as she did,\" yielding \"as indeed she did.\" The predicted edit captures the same core modification by changing \"as\" to \"as indeed,\" which effectively represents inserting \"indeed\" immediately after \"as.\" Although the prediction provides less surrounding context (it omits \"she did\"), it still captures the essential intended change (the insertion of \"indeed\" in that location)."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length",
"corrected_text": "length,",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\nText Content Accuracy: The ground truth change is inserting a comma after the word \"length\" in the phrase \"length said\" to make it \"length, said.\" The prediction captures this same core change by editing \"length\" to \"length,\" (adding the comma in the correct location). Although it includes less surrounding context (it does not include \"said\"), it still reflects the essential intended punctuation correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes the phrase \"I hope so\" to \"I hoped so\", which is fundamentally the tense change of \"hope\" \u2192 \"hoped\" within that phrase. The predicted edit captures exactly this core change (\"hope\" \u2192 \"hoped\"), just with less surrounding context. This still reflects the same intended correction."
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:57.526065",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:23:19.186886",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart",
"corrected_text": "heart,",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are of type \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma immediately after the word \"heart\" in the phrase \"heart so\" \u2192 \"heart, so\". The predicted edit \"heart\" \u2192 \"heart,\" captures the same core punctuation change (adding a comma after \"heart\"). Although it includes less surrounding context (it omits the following word \"so\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "generous,",
"corrected_text": "generous",
"observed_line_number": 22,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is removing the comma after \"generous\" in the phrase \"generous, Mr.\" to make it \"generous Mr.\" The predicted edit removes the comma after \"generous\" (\"generous,\" \u2192 \"generous\"). Although it does not include the following \"Mr.\", it captures the core intended punctuation change (deleting the comma after \"generous\"). Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "as",
"corrected_text": "as indeed",
"observed_line_number": 27,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes \"as she did\" to \"as indeed she did,\" i.e., it inserts the word \"indeed\" immediately after \"as.\" The predicted edit changes \"as\" to \"as indeed,\" which captures the same core insertion (adding \"indeed\" after \"as\"). Although the prediction includes a smaller text span and omits the surrounding context (\"she did\"), it still represents the essential intended change."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length",
"corrected_text": "length,",
"observed_line_number": 31,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after the word \"length\" (changing \"length said\" to \"length, said\"). The prediction changes \"length\" to \"length,\", which captures the same core punctuation insertion at the correct location. Although it includes less surrounding context (omits \"said\"), it still represents the essential intended edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "hope",
"corrected_text": "hoped",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the phrase \"I hope so\" to \"I hoped so,\" which core-wise is the verb tense change \"hope\" \u2192 \"hoped.\" The prediction replaces \"hope\" with \"hoped\" without the surrounding words, but it captures the essential intended correction. Additional context is not required as long as the core change is correct."
}
],
"file_id": "009"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:56.681631",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:21.414344",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change inserts a comma after \"hers\" in the phrase \"hers and\" \u2192 \"hers, and.\" The prediction captures the same core punctuation insertion by changing \"hers\" \u2192 \"hers,\" (even though it omits the following word \"and\"). This still reflects the essential intended edit: adding a comma immediately after \"hers.\""
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" in the phrase \"hers matters\" \u2192 \"hers, matters.\" The predicted edit \"hers\" \u2192 \"hers,\" captures the same essential punctuation insertion (comma after \"hers\"). The omission of the following word \"matters\" is acceptable because it is additional context, not part of the core change.\n\nTherefore, the prediction correctly captures the ground truth\u2019s intended edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"moment\" in the phrase \"moment with\" \u2192 \"moment, with.\" The predicted edit \"moment\" \u2192 \"moment,\" captures the same core change (adding a comma immediately after \"moment\"). Although it includes less surrounding context, it correctly reflects the intended punctuation insertion."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful",
"corrected_text": "beautiful,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"beautiful\" in the phrase \"beautiful that\" \u2192 \"beautiful, that.\" The predicted edit \"beautiful\" \u2192 \"beautiful,\" captures the same essential punctuation insertion (a comma immediately after \"beautiful\"). Although it includes less surrounding context (omits \"that\"), it still reflects the core intended edit."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both identify the same original text (\"they\") and the same correction (\"They\"), capturing the identical core change (capitalizing the initial letter). No discrepancy in the intended modification."
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:00.448282",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:20:43.239405",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" in the phrase \"hers and\" \u2192 \"hers, and.\" The prediction captures the same core punctuation change by editing \"hers\" \u2192 \"hers,\" even though it omits the following \"and\" from the snippet. Since the essential intended correction (adding the comma after \"hers\") is correctly represented, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates inserting a comma after \"hers\" in the phrase \"hers matters\" \u2192 \"hers, matters.\" The prediction captures the same core punctuation change by editing \"hers\" \u2192 \"hers,\". Although it omits the following word \"matters,\" it still represents the essential intended edit (adding a comma after \"hers\")."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after the word \"moment\" (\"moment with\" \u2192 \"moment, with\"). The prediction also inserts a comma after \"moment\" (\"moment\" \u2192 \"moment,\"). Although the prediction uses a smaller text span and does not include the following word \"with\", it captures the core intended punctuation change\u2014adding the comma after \"moment\". Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful",
"corrected_text": "beautiful,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"beautiful\" in the phrase \"beautiful that\" \u2192 \"beautiful, that\". The predicted edit \"beautiful\" \u2192 \"beautiful,\" captures the same core punctuation change (adding a comma immediately after \"beautiful\"). The omission of the following word \"that\" is acceptable because it is just surrounding context; the essential intended edit (comma insertion after \"beautiful\") is correctly represented."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"they\" \u2192 \"They\". There is no difference in original or corrected text, so the essential intended edit is captured correctly."
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:03.832665",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:23:00.422336",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \u201chers\u201d in the phrase \u201chers and\u201d \u2192 \u201chers, and\u201d. The prediction captures the same core punctuation change by editing \u201chers\u201d \u2192 \u201chers,\u201d. Although it omits the following word \u201cand\u201d from the span, it still reflects the essential intended correction (adding the comma after \u201chers\u201d)."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers",
"corrected_text": "hers,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma immediately after the word \"hers\" in the phrase \"hers matters\" \u2192 \"hers, matters\". The predicted edit \"hers\" \u2192 \"hers,\" captures this same core punctuation change (adding a comma after \"hers\"). Although it omits the following word \"matters\" from the original/corrected text span, it still reflects the essential intended correction (comma insertion after \"hers\")."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment",
"corrected_text": "moment,",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"moment\" in the phrase \"moment with\" \u2192 \"moment, with\". The predicted edit \"moment\" \u2192 \"moment,\" captures the same core punctuation change (adding the comma after \"moment\"). Although the prediction includes less surrounding context (it does not include \"with\"), the essential intended correction is accurately represented."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "beautiful",
"corrected_text": "beautiful,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after the word \"beautiful\" in the phrase \"beautiful that\" \u2192 \"beautiful, that.\" The predicted edit \"beautiful\" \u2192 \"beautiful,\" captures the same core punctuation change (adding a comma immediately after \"beautiful\"). Although it includes less surrounding context (omits \"that\"), it correctly reflects the essential intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same transformation from \"they\" to \"They\", capturing the identical core capitalization change. No discrepancy in original or corrected text."
}
],
"file_id": "010"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:00.354492",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:18:33.485941",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that,",
"corrected_text": "that",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches.\n\n2) Text Content Accuracy: The ground truth change is removing the comma after \"that\" in the phrase \"that, in\" \u2192 \"that in\". The prediction removes the comma after \"that\" as well (\"that,\" \u2192 \"that\"). Although it omits the following word \"in\" as context, it still captures the core intended correction: deletion of the comma after \"that\". Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy,",
"corrected_text": "fancy",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change removes the comma after \u201cfancy\u201d in the phrase \u201cfancy, whatever,\u201d yielding \u201cfancy whatever.\u201d The prediction specifies \u201cfancy,\u201d \u2192 \u201cfancy,\u201d which captures the same core action: deleting the comma after \u201cfancy.\u201d Although the prediction includes less surrounding context (it omits \u201cwhatever\u201d), it still correctly represents the essential punctuation correction intended by the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you!",
"corrected_text": "you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the ending punctuation from an exclamation point to a comma in the phrase \"Thank you!\" \u2192 \"Thank you,\". The prediction captures the same core change (\"!\" \u2192 \",\") but with less surrounding context, editing \"you!\" \u2192 \"you,\". This still reflects the essential intended correction to replace the exclamation point with a comma at the end of the phrase."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after the word \"shore\" (turning \"shore and\" into \"shore, and\"). The predicted edit changes \"shore\" to \"shore,\", which captures the same essential punctuation insertion at the correct location. Although the prediction includes less surrounding context (it omits \"and\"), it still represents the core change accurately."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both change the original text \"that\" to the corrected text \"than\". This captures the identical core correction with no discrepancy.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:25:58.853909",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:20:49.536260",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that,",
"corrected_text": "that",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth removes a comma after \"that\" in the sequence \"that, in\" \u2192 \"that in.\" The predicted edit \"that,\" \u2192 \"that\" captures the same core action: deleting the comma after \"that.\" Although the prediction omits the following word \"in\" (less surrounding context), it still reflects the essential intended punctuation change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy,",
"corrected_text": "fancy",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" (removing a comma). The predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"fancy, whatever\" to \"fancy whatever\", whose core action is deleting the comma after \"fancy\". The prediction changes \"fancy,\" to \"fancy\", which captures that same essential punctuation removal, just with less surrounding context (it omits \"whatever\"). Since the core intended change (removing the comma after \"fancy\") is correctly represented, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\nText Content Accuracy: Both specify the same core punctuation change, converting \"Thank you!\" to \"Thank you,\". There is no discrepancy in original or corrected text. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to insert a comma after \"shore\" in the phrase \"shore and\" \u2192 \"shore, and\". The prediction \"shore\" \u2192 \"shore,\" captures the same core change (adding a comma immediately after \"shore\"). Although it omits the following word \"and\" from the context, the essential punctuation insertion is correctly represented."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the original text \"that\" with the corrected text \"than\". The core intended change is identical with no deviation. Therefore, the prediction correctly captures the ground truth edit."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:10.345400",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:23:03.515639",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that,",
"corrected_text": "that",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change removes the comma after \"that\" in the phrase \"that, in\" \u2192 \"that in\". The prediction removes the comma after \"that\" as well: \"that,\" \u2192 \"that\". Although the prediction omits the following word \"in\" as context, it still captures the core intended correction (deleting the comma after \"that\"). Under the rubric, missing or extra surrounding context is acceptable as long as the essential change is correctly represented."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy,",
"corrected_text": "fancy",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth removes the comma after \"fancy\" in the phrase \"fancy, whatever\" to make it \"fancy whatever.\" The prediction removes the comma from \"fancy,\" to \"fancy.\" Although the prediction includes less surrounding context (it omits \"whatever\"), it captures the core intended change: deleting the comma after \"fancy.\" Therefore, the essential edit is correctly represented."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you!",
"corrected_text": "you,",
"observed_line_number": 17,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the terminal punctuation from an exclamation point to a comma in \"Thank you!\" \u2192 \"Thank you,\". The predicted edit captures the same core change by changing \"you!\" \u2192 \"you,\". Although it includes less surrounding context (omits \"Thank\"), it correctly represents the essential punctuation replacement at the end of the phrase.\n\nTherefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"shore and\" to \"shore, and,\" which is specifically the insertion of a comma after \"shore\". The predicted edit changes \"shore\" to \"shore,\", also inserting a comma after \"shore\". Although the prediction includes less surrounding context (it doesn\u2019t include \"and\"), it captures the same core intended punctuation change (adding the comma after \"shore\")."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 33,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the original text \"that\" with the corrected text \"than\". The core intended change is identical. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:31.021334",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:19:13.835109",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XXIX",
"corrected_text": "XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth adds a period to the end of the chapter heading (\"Chapter XXIX\" \u2192 \"Chapter XXIX.\"). The prediction adds the same period to the end of the numeral portion (\"XXIX\" \u2192 \"XXIX.\"). Although the prediction omits the preceding word \"Chapter\" from the span of text it quotes, it captures the core intended change: inserting a final period after \"XXIX\" in the heading. Additional context is not required as long as the essential punctuation change is represented correctly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same change from \"city\" to \"City\", capturing the identical core correction (capitalizing the initial letter). Therefore, the predicted edit fully matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both indicate the same substitution, changing \"of\" to \"to\". Since the core change is identical, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: replacing the comma after \"them\" (\"them,\") with a semicolon (\"them;\"). The original_text and corrected_text align perfectly with the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. Text Content Accuracy: Both specify the same change from \"existence\" to \"existence,\", correctly capturing the core intended punctuation insertion (adding a comma). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same original text (\"it,\") and the same corrected text (\"it;\"), capturing the identical core change of replacing a comma with a semicolon after \"it\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: adding a comma after the word \"motionless\" (\"motionless\" \u2192 \"motionless,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit fully captures the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"stand-still,\") and the same correction (\"stand-still;\"), capturing the identical core change of replacing a comma with a semicolon. Therefore the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 8,
"expected_edit_num": 8,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. Text Content Accuracy: Both specify the same core change, adding a comma after \"view\" (\"view\" \u2192 \"view,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 9,
"expected_edit_num": 9,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n\nText Content Accuracy: Both specify the same core punctuation change from \"existence,\" to \"existence;\". The original_text and corrected_text align perfectly with the ground truth intention.\n\nTherefore, the predicted edit correctly captures the intended correction."
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:38.364311",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:21:17.192236",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XXIX",
"corrected_text": "XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is adding a period at the end of the chapter heading: \"Chapter XXIX\" \u2192 \"Chapter XXIX.\". The prediction captures the same core punctuation change by adding a period after the roman numeral: \"XXIX\" \u2192 \"XXIX.\". Although it includes less surrounding context (omits the word \"Chapter\"), it still represents the essential intended correction\u2014adding the terminal period after XXIX in the heading."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit have type \"capitalization\", so the type matches exactly. Text Content Accuracy: Both specify changing \"city\" to \"City\", which captures the identical core correction (capitalizing the first letter). Therefore, the prediction matches the ground truth\u2019s intended edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\nText Content Accuracy: Both identify the same original text (\"of\") being replaced with the same corrected text (\"to\"). This captures the core intended change with no discrepancy.\nTherefore, the predicted edit correctly matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change: replacing \"them,\" with \"them;\". The original_text and corrected_text are identical between ground truth and prediction, so the intended edit is fully captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly. Text Content Accuracy: Both specify the identical transformation from \"existence\" to \"existence,\" (adding a trailing comma). The core intended edit is captured with no discrepancy."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"motionless\" (\"motionless\" \u2192 \"motionless,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"view\" to \"view,\" (adding a comma). The core punctuation modification is identical. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:41.580966",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:23:10.265649",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "XXIX",
"corrected_text": "XXIX.",
"observed_line_number": 0,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes \"Chapter XXIX\" to \"Chapter XXIX.\"\u2014i.e., it adds a period at the end of the chapter heading. The predicted edit changes \"XXIX\" to \"XXIX.\", which captures the same core punctuation change (adding a terminal period after the Roman numeral). The prediction simply uses a narrower span of the original text (omits the preceding word \"Chapter\"), but it still reflects the intended correction at the correct location (end of the heading)."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the type matches exactly.\nText Content Accuracy: Both specify the same transformation from \"city\" to \"City\", capturing the identical core change (capitalizing the initial letter). No discrepancies or missing/extra content affect the essential edit."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": 5,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is a \"replacement\" and the predicted edit type is also \"replacement\", so the type matches exactly.\n\nText Content Accuracy: The ground truth replaces the word \"of\" with \"to\". The prediction specifies the same original text (\"of\") and the same corrected text (\"to\"). This captures the core intended change with no discrepancy.\n\nTherefore, both criteria are satisfied."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": 10,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: replacing \"them,\" with \"them;\". There is no discrepancy in original or corrected text. Therefore, the prediction captures the ground truth edit intention \u043f\u043e\u043b\u043d\u043e\u0441\u0442\u044c\u044e."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": 14,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change, adding a comma after \"existence\" (\"existence\" \u2192 \"existence,\"). There is no discrepancy in the original or corrected text. Therefore, the predicted edit correctly captures the intended correction."
},
{
"observed_edit_num": 5,
"expected_edit_num": 5,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\nText Content Accuracy: Both ground truth and prediction change \"motionless\" to \"motionless,\" by adding a comma immediately after the word. The core punctuation change is identical.\nTherefore, the prediction correctly captures the intended edit."
},
{
"observed_edit_num": 6,
"expected_edit_num": 6,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": 16,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted type is \"punctuation,\" which exactly matches the ground truth. Text Content Accuracy: Both ground truth and prediction change \"view\" to \"view,\" by adding a comma immediately after the word. The core intended change is captured exactly. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 7,
"expected_edit_num": 7,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence;",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth changes the punctuation after the word from a comma to a semicolon (\"existence,\" \u2192 \"existence;\"). The prediction also results in \"existence;\" as the corrected text. Although the predicted original_text omits the comma (\"existence\" instead of \"existence,\"), it still captures the core intended change: the punctuation following \"existence\" should be a semicolon in the corrected version. Therefore, the essential edit intent is correctly represented."
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Dreaming",
"corrected_text": "Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 9,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "it",
"corrected_text": "it;",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 10,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand-still",
"corrected_text": "stand-still;",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 11,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": 25,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:38.494156",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:19:27.012638",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core punctuation change: replacing the colon in \"Clennam:\" with a semicolon to form \"Clennam;\". The original and corrected text match the ground truth intention exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly. Text Content Accuracy: Both specify the same target text (\"him\") with the intention to italicize it; there is no discrepancy in original/corrected text content. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now",
"corrected_text": "now,",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"now\" in the phrase \"now I\" \u2192 \"now, I\". The prediction captures the same core punctuation insertion by changing \"now\" \u2192 \"now,\". Although it omits the following word \"I\" in the span, it correctly represents the essential intended edit (adding a comma after \"now\")."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both replace the same original text \"authorize\" with the same corrected text \"authorise\". The core change (American to British spelling) is identical. Therefore, the predicted edit captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now",
"corrected_text": "now,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "that\u2014",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you",
"corrected_text": "you,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:43.787920",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:21:52.344416",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: The predicted change matches the ground truth core correction, changing \"Clennam:\" to \"Clennam;\" with identical original and corrected text. Therefore, it captures the intended punctuation change exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify the edit type as \"italicize\", so the type matches exactly.\nText Content Accuracy: Both identify the same target text \"him\" with no change in wording (only formatting), which correctly captures the intended italicization of the word. Therefore, the prediction matches the ground truth's core change."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now",
"corrected_text": "now,",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after \"now\" in the phrase \"now I\" (\"now I\" \u2192 \"now, I\"). The predicted edit (\"now\" \u2192 \"now,\") captures the same essential punctuation change\u2014adding a comma immediately after \"now.\" The prediction simply provides less surrounding context (it omits the following \"I\"), which is acceptable since the core intended change is still correctly represented."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and predicted edit are of type \"replacement\", so the edit type matches exactly.\nText Content Accuracy: Both specify replacing \"authorize\" with \"authorise\". The core change (American spelling to British spelling) is identical. Therefore, the prediction captures the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now",
"corrected_text": "now,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that boiling-over",
"corrected_text": "that\u2014boiling-over",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you",
"corrected_text": "you,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:40.154289",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:24:11.486431",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The predicted edit type is \"punctuation,\" which exactly matches the ground truth.\n\nText Content Accuracy: Both ground truth and prediction change \"Clennam:\" to \"Clennam;\"\u2014the same core punctuation substitution (colon to semicolon) on the same text. Therefore, the prediction captures the intended correction exactly."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both specify the same target text \"him\" with no change in wording (only formatting via italics). The prediction captures the same core intended change as the ground truth."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now",
"corrected_text": "now,",
"observed_line_number": 34,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction are labeled as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"now\" in the phrase \"now I\" \u2192 \"now, I.\" The prediction changes \"now\" \u2192 \"now,\", which captures the same essential punctuation change (adding a comma immediately after \"now\"). Although it omits the following word \"I\" as context, it still reflects the core intended edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 37,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly. Text Content Accuracy: Both replace the same original text \"authorize\" with the same corrected text \"authorise\", capturing the identical core change (American to British spelling). Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "snorts",
"corrected_text": "of snorts",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "now",
"corrected_text": "now,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest",
"corrected_text": "driest",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "that\u2014",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 8,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "you",
"corrected_text": "you,",
"observed_line_number": 37,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of the dryest snorts.",
"corrected_text": "of the driest of snorts.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:32.333051",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:19:40.422957",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For",
"corrected_text": "For,",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks.\" The prediction captures the same core change by editing \"For\" \u2192 \"For,\". Although it omits the following words (\"Mr. Pancks\"), it still correctly represents the essential punctuation insertion at the correct location."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is to insert a comma after the word \"quickly\" in the phrase \"quickly that\" \u2192 \"quickly, that.\" The predicted edit \"quickly\" \u2192 \"quickly,\" captures the same core punctuation insertion (adding a comma immediately after \"quickly\"). Although it includes less surrounding context (it omits \"that\"), it still reflects the essential intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "me",
"corrected_text": "me a",
"observed_line_number": 21,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "me,",
"corrected_text": "me a,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:37.087658",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:21:43.165749",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For",
"corrected_text": "For,",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the edit type matches exactly.\n2. Text Content Accuracy: The ground truth inserts a comma after \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks\". The predicted edit captures the same core change by changing \"For\" \u2192 \"For,\". Although it omits the surrounding words (\"Mr. Pancks\"), it correctly represents the essential punctuation insertion identified in the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth change inserts a comma after \"quickly\" in the phrase \"quickly that\" \u2192 \"quickly, that\". The prediction specifies \"quickly\" \u2192 \"quickly,\", which captures the same core punctuation change (adding a comma immediately after \"quickly\"). The prediction omits the following word \"that\", but additional context is optional as long as the essential edit is correct, which it is here."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "me",
"corrected_text": "me a",
"observed_line_number": 21,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as an \"insertion,\" so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth indicates inserting the word \"a\" after \"me\" (changing \"me,\" to \"me a,\"). The prediction also captures the same core insertion (\"me\" \u2192 \"me a\"). Although the prediction omits the comma shown in the ground truth, it still reflects the essential intended change\u2014adding \"a\" after \"me\"\u2014and does not contradict that intent. Therefore, the text content matches the ground truth\u2019s core edit."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed-room",
"corrected_text": "bedroom",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
},
{
"model_name": "or_openai_gpt_6_astra",
"date": "2026-09-06T04:26:37.082721",
"annotator": "GPT-6 Astra (OR)",
"annotation_date": "2026-09-06T04:24:01.606436",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "For",
"corrected_text": "For,",
"observed_line_number": 2,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The predicted edit type is \"punctuation,\" which matches the ground truth exactly.\n\n2. Text Content Accuracy: The ground truth change is inserting a comma after the word \"For\" in the phrase \"For Mr. Pancks\" \u2192 \"For, Mr. Pancks.\" The prediction captures this core change by editing \"For\" \u2192 \"For,\". Although it omits the surrounding words (\"Mr. Pancks\"), it still correctly represents the essential punctuation insertion at the correct location (immediately after \"For\")."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "quickly",
"corrected_text": "quickly,",
"observed_line_number": 11,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after \"quickly\" in the phrase \"quickly that\" \u2192 \"quickly, that.\" The predicted edit \"quickly\" \u2192 \"quickly,\" captures the same essential punctuation change (adding a comma immediately after \"quickly\"). The prediction uses a shorter text span but correctly represents the core intended correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "insertion",
"original_text": "me",
"corrected_text": "me a",
"observed_line_number": 21,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as an \"insertion,\" so the type matches.\n\n2) Text Content Accuracy: The ground truth change is essentially inserting the word \"a\" after \"me\" (turning \"me,\" into \"me a,\"). The prediction also captures this core change by transforming \"me\" into \"me a.\" The missing comma in the predicted original/corrected text appears to be a difference in how much surrounding punctuation/context was included, not a failure to capture the intended insertion (the comma is not being added or removed in the ground truth; it remains). Therefore, the prediction matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "bed room",
"corrected_text": "bed-room",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "bed-room",
"corrected_text": "bed room",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "014"
}
],
"vendor": "OpenAI",
"release_date": "2026-09-03",
"release_source": "https://developers.openai.com/api/docs/models/gpt-6-astra",
"release_notes": "OpenAI's GPT-6 flagship model. Benchmark configuration uses high reasoning effort."
},
{
"model_name": "GPT-o4 Mini",
"model_id": "gpt_o4_mini",
"date": "2026-02-22T15:37:22.828009",
"shots": 2,
"config": {
"model_id": "gpt_o4_mini",
"display_name": "GPT-o4 Mini",
"shots": 2,
"temperature": 0.0,
"date": "2025-04-16",
"notes": "Benchmark run with 2-shot learning"
},
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:17:55.466078",
"file_results": [
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:01.956623",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:17:55.466078",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"howling over\") and the same corrected text (\"howling, over\"), capturing the identical core change (inserting a comma after \"howling\"). Ignoring line numbers, the prediction fully matches the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit type as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"yonder to-day\") and the same corrected text (\"yonder, to-day\"), capturing the identical core change: inserting a comma after \"yonder\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:34:58.204701",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:46:15.341359",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both apply the same core change: adding a period after \"Fellow Travellers\" (\"Fellow Travellers\" \u2192 \"Fellow Travellers.\"). The original and corrected texts match the ground truth\u2019s intended punctuation change."
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:06.970153",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:05:28.906511",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "Fellow Travellers",
"corrected_text": "Fellow Travellers.",
"observed_line_number": 1,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1. Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both edits change \"Fellow Travellers\" to \"Fellow Travellers.\" by adding a period at the end. This captures the same core punctuation correction with identical original and corrected text. Line number differences are ignored as instructed."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "sir,is",
"corrected_text": "sir, is",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "howling,over",
"corrected_text": "howling, over",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "to\u2011day,sir",
"corrected_text": "to\u2011day, sir",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "yonder to-day",
"corrected_text": "yonder, to-day",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "pockets and",
"corrected_text": "pockets, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "howling over",
"corrected_text": "howling, over",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "Sir",
"corrected_text": "sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "003"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:34:54.090136",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:19:02.434061",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "why",
"corrected_text": "Why",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ",",
"corrected_text": "?",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:02.598942",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:47:27.727639",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "that",
"corrected_text": "That",
"observed_line_number": 28,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stones",
"corrected_text": "stones?",
"observed_line_number": 29,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:13.351791",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:06:58.286078",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "No sir,",
"corrected_text": "No, sir,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "She had no belief",
"corrected_text": "she had no belief",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "No sir",
"corrected_text": "No, sir",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "why did he dine",
"corrected_text": "Why did he dine",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "cold stones!",
"corrected_text": "cold stones?",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "004"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:20.317052",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:20:28.947834",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit, so the edit type matches exactly. Text Content Accuracy: Both replace the same original text \"said Clennam\" with the same corrected text \"asked Clennam\". The core change is identical, with no discrepancies in wording or scope. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "? If",
"corrected_text": ", if",
"observed_line_number": 36,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "if he",
"corrected_text": "If he",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:31.614505",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:48:42.552220",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 36,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"capitalization\", so the edit type matches exactly.\n\n2) Text Content Accuracy: The ground truth capitalizes the initial word in the phrase \"if he\" \u2192 \"If he\". The prediction captures the same core change by capitalizing \"if\" \u2192 \"If\". Although the prediction includes less surrounding context (it does not include \"he\"), it still represents the essential intended correction: capitalizing \"if\" at the start of the phrase/sentence. Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "said",
"corrected_text": "asked",
"observed_line_number": 39,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both the ground truth and the prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2. Text Content Accuracy: The ground truth changes the phrase \"said Clennam\" to \"asked Clennam\", where the essential correction is replacing \"said\" with \"asked\". The predicted edit replaces \"said\" with \"asked\" without including the following word \"Clennam\". This omission is acceptable because it still captures the core intended change (said \u2192 asked) identified in the ground truth."
}
],
"file_id": "005"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:40.023462",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:08:12.573030",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "if",
"corrected_text": "If",
"observed_line_number": 37,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is capitalizing the word \"if\" at the start of the phrase \"if he\" \u2192 \"If he\". The prediction captures the same core change by capitalizing \"if\" \u2192 \"If\". Although it provides less surrounding context (omitting \"he\"), it still reflects the essential intended correction: capitalizing \"if\". Therefore, the text content matches the ground truth\u2019s intention."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "sad",
"corrected_text": "asked",
"observed_line_number": 40,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "said Clennam",
"corrected_text": "asked Clennam",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "005"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:34.926865",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:21:43.997744",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the change as a \"punctuation\" edit, so the type matches exactly.\n2) Text Content Accuracy: Both specify the same original text (\"materials costing\") and the same corrected text (\"materials, costing\"), capturing the identical core change (inserting a comma after \"materials\").\nLine number differences are ignored as instructed, so the prediction correctly matches the ground truth intent."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and came",
"corrected_text": "about, and came",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "road",
"corrected_text": "high\u2011road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:47.824415",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:50:09.186715",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high\u2011road",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The predicted type is \"punctuation,\" which matches the ground truth exactly.\n\n2) Text Content Accuracy: The ground truth change is to hyphenate \"high road\" \u2192 \"high-road.\" The prediction changes \"high road\" \u2192 \"high\u2011road\" using a non-breaking hyphen (U+2011) instead of a standard hyphen-minus (-). Despite the different hyphen character, the core intended edit\u2014adding a hyphen between \"high\" and \"road\"\u2014is correctly captured."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials costing",
"corrected_text": "materials, costing",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same transformation: inserting a comma after \"materials\" in the phrase \"materials costing\" to produce \"materials, costing\". The core change is identical, with no discrepancies in original or corrected text."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edit have the type \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both specify the same core punctuation change, inserting a comma after \"about\" (\"about and\" \u2192 \"about, and\"). There is no discrepancy in original or corrected text. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "said 'Clennam",
"corrected_text": "said, 'Clennam",
"observed_line_number": 25,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both the ground truth and the prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth intends to add a comma after the word \"said\" (\"said\" \u2192 \"said,\"). The predicted edit also inserts a comma immediately after \"said\" (\"said 'Clennam\" \u2192 \"said, 'Clennam\"). Although the prediction includes additional surrounding context (the following quoted name), the core punctuation change is the same and correctly captured."
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said when",
"corrected_text": "said, when",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:53.969105",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:09:28.352933",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "materials",
"corrected_text": "materials,",
"observed_line_number": 3,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth inserts a comma after the word \"materials\" in the phrase \"materials costing\" \u2192 \"materials, costing.\" The predicted edit \"materials\" \u2192 \"materials,\" captures the same core change (adding a comma immediately after \"materials\"). Although it omits the following word \"costing,\" it still represents the essential intended punctuation correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "about and",
"corrected_text": "about, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same change from \"about and\" to \"about, and,\" inserting a comma after \"about.\" The core intended punctuation correction is captured identically."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "earnestly telling",
"corrected_text": "earnestly, telling",
"observed_line_number": 18,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"punctuation\" and the predicted edit type is also \"punctuation\", so the type matches exactly. Text Content Accuracy: Both ground truth and prediction change \"earnestly telling\" to \"earnestly, telling\" by inserting a comma after \"earnestly\". The original and corrected text match the core intended change precisely. Therefore, the predicted edit correctly captures the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.6,
"fp": 0.2,
"fn": 0.2,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": 25,
"line_diff": 2,
"line_number_penalty": 0.4,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \u201csaid\u201d in the phrase \u201csaid when\u201d \u2192 \u201csaid, when.\u201d The predicted edit \u201csaid\u201d \u2192 \u201csaid,\u201d captures the same essential punctuation insertion at the same location (after \u201csaid\u201d). Although it includes less surrounding context (it omits \u201cwhen\u201d), it still reflects the core intended correction."
},
{
"observed_edit_num": 4,
"expected_edit_num": 4,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Clennam in",
"corrected_text": "Clennam, in",
"observed_line_number": 29,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both edits change \"Clennam in\" to \"Clennam, in\" by inserting a comma after \"Clennam\". The original and corrected texts match the ground truth core change exactly, with no deviation in content."
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "highroad",
"corrected_text": "high\u2011road",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "association did",
"corrected_text": "association, did",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "high road",
"corrected_text": "high-road",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "association, did",
"corrected_text": "association did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "said",
"corrected_text": "said,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "006"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:35:51.290448",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:22:58.232796",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter I,",
"corrected_text": "Chapter I.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:02.825642",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:51:18.679677",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": ".",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": ",",
"corrected_text": ".",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:05.607788",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:10:31.628797",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": ".",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Sun and Shadow",
"corrected_text": "Sun and Shadow.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "007"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:17.259917",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:24:17.324875",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "sodden",
"corrected_text": "sodden",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 2,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ",",
"corrected_text": "",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:22.349893",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:52:44.348842",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar\u2011trees",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons,was",
"corrected_text": "Chalons, was",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "soddenwith",
"corrected_text": "sodden with",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:18.568438",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:11:51.893866",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n\n2) Text Content Accuracy: Both edits replace the same original text \"Saone\" with the same corrected text \"Sa\u00f4ne\" (adding the circumflex over the 'o'). The core intended change is identical.\n\nTherefore, the predicted edit correctly captures the ground truth edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter\u00a0XI",
"corrected_text": "Chapter\u00a0XI.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar\u2010trees",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Let Loose",
"corrected_text": "Let Loose.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "dull autumn",
"corrected_text": "dull, autumn",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "Saone",
"corrected_text": "Sa\u00f4ne",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "man slowly",
"corrected_text": "man, slowly",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chalons was",
"corrected_text": "Chalons, was",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "sodden",
"corrected_text": "soddened",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "poplar trees",
"corrected_text": "poplar-trees,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "008"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:14.964643",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:25:42.782516",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so much",
"corrected_text": "heart, so much",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "smiling;",
"corrected_text": "smiling,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "I hoped so",
"corrected_text": "I hoped so",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:16.137909",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:54:27.079886",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "confidentially",
"corrected_text": "indeed",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "so;",
"corrected_text": "so,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:23.314426",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:13:18.727999",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "indeed",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ";",
"corrected_text": ",",
"observed_line_number": 14,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "That's well! I hoped so; I am not disappointed.",
"corrected_text": "I hoped so",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "Mr.",
"corrected_text": "",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "generous, Mr.",
"corrected_text": "generous Mr.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "insertion",
"original_text": "as she did",
"corrected_text": "as indeed she did",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "length said",
"corrected_text": "length, said",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "I hope so",
"corrected_text": "I hoped so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "heart so",
"corrected_text": "heart, so",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "009"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:34.979061",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:26:59.925449",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and thanked",
"corrected_text": "hers, and thanked",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the type matches exactly.\n\n2) Text Content Accuracy: The ground truth change is inserting a comma after \"hers\" in the phrase \"hers and\" \u2192 \"hers, and\". The prediction expands the span to include an extra following word (\"thanked\"), but it still performs the same core punctuation change: \"hers and thanked\" \u2192 \"hers, and thanked\". This correctly captures the intended edit, with permissible additional context."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation,\" so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change\u2014adding a comma after \"hers\"\u2014with identical original_text (\"hers matters\") and corrected_text (\"hers, matters\"). The essential punctuation correction is captured exactly."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 22,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both edits change the same phrase from \"moment with\" to \"moment, with\" by inserting a comma after \"moment\". This captures the core intended punctuation correction with identical original and corrected text."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 31,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "Edit Type Accuracy: The predicted edit type is \"capitalization,\" which exactly matches the ground truth type.\n\nText Content Accuracy: The ground truth changes \"they\" to \"They\". The prediction specifies the same original text (\"they\") and the same corrected text (\"They\"), capturing the identical core capitalization change. Line number differences are ignored as instructed."
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:29.008649",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:55:51.861346",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "from hers and",
"corrected_text": "from hers, and",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "on hers matters",
"corrected_text": "on hers, matters",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 16,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:42.701514",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:14:44.023082",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers and",
"corrected_text": "hers, and",
"observed_line_number": 13,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1. Edit Type Accuracy: Both ground truth and predicted edit are labeled as \"punctuation\", so the type matches exactly.\n2. Text Content Accuracy: Both edits change \"hers and\" to \"hers, and\" by inserting a comma after \"hers\". The core punctuation change is identical in the prediction and the ground truth."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "hers matters",
"corrected_text": "hers, matters",
"observed_line_number": 15,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"punctuation\", so the type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: inserting a comma after \"hers\" in the phrase \"hers matters\" to become \"hers, matters\". The original and corrected text strings match exactly, capturing the intended punctuation correction."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "moment with",
"corrected_text": "moment, with",
"observed_line_number": 23,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: The ground truth type is \"punctuation\" and the predicted type is also \"punctuation\", so the types match exactly.\n\n2) Text Content Accuracy: Both specify the same original text (\"moment with\") and the same corrected text (\"moment, with\"), capturing the identical core change: insertion of a comma after \"moment\". Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": 3,
"expected_edit_num": 3,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "they",
"corrected_text": "They",
"observed_line_number": 32,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: Both ground truth and prediction label the change as a \"capitalization\" edit, so the type matches exactly. Text Content Accuracy: Both specify changing the lowercase word \"they\" to the capitalized form \"They\". This captures the core intended correction with identical original and corrected text. Therefore, the predicted edit matches the ground truth intention."
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "beautiful that",
"corrected_text": "beautiful, that",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "010"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:49.990052",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:28:09.726331",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 11,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 18,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 20,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 23,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "that",
"corrected_text": "that,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy",
"corrected_text": "fancy,",
"observed_line_number": 8,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:43.600493",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:57:10.386002",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "in",
"corrected_text": "",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "fancy",
"corrected_text": "fancy,",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore",
"corrected_text": "shore,",
"observed_line_number": 13,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 15,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": 17,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:45.859047",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:15:58.044025",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "that",
"corrected_text": "",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": "in",
"corrected_text": "",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Thank you?",
"corrected_text": "Thank you,",
"observed_line_number": 6,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": 9,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": 10,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "that once",
"corrected_text": "than once",
"observed_line_number": 12,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 0,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that, in",
"corrected_text": "that in",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "fancy, whatever",
"corrected_text": "fancy whatever",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Thank you!",
"corrected_text": "Thank you,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "shore and",
"corrected_text": "shore, and",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "within doors",
"corrected_text": "within-doors",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "that",
"corrected_text": "than",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "011"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:53.098314",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:29:22.027822",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 1,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the edit as \"capitalization\", so the type matches exactly. 2) Text Content Accuracy: Both specify the same core change from \"city\" to \"City\". There are no differences in the original or corrected text, so the prediction fully captures the intended correction."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs. Flintwinch goes on Dreaming",
"corrected_text": "Mrs. Flintwinch goes on Dreaming.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to",
"observed_line_number": 4,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:37:03.718928",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:58:28.717548",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and predicted edits are labeled as \"capitalization\", so the edit type matches exactly.\n2) Text Content Accuracy: Both specify the same core change: \"city\" is corrected to \"City\". This captures the essential intended correction. Line number differences are to be ignored, so they do not affect correctness."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ",",
"corrected_text": "",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": ";",
"corrected_text": ",",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "capitalization",
"original_text": "Is",
"corrected_text": "is",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "deletion",
"original_text": ".",
"corrected_text": "",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": "to",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "",
"corrected_text": " ",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:36:57.784659",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T22:17:50.716306",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "capitalization",
"original_text": "city",
"corrected_text": "City",
"observed_line_number": 2,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: The predicted edit type is \"capitalization,\" which matches the ground truth exactly.\n2) Text Content Accuracy: Both edits change the same word from \"city\" to \"City,\" capturing the identical core correction (capitalizing the initial letter). Since line numbers are to be ignored, there is no discrepancy affecting the intended edit."
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Chapter xxix",
"corrected_text": "Chapter xxix.",
"observed_line_number": 0,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 2,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "Mrs.\u00a0Flintwinch goes on Dreaming",
"corrected_text": "Mrs.\u00a0Flintwinch goes on Dreaming.",
"observed_line_number": 1,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 3,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "replacement",
"original_text": "busy\u00a0existence,",
"corrected_text": "busy\u00a0existence;",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 4,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "stand\u2011still,",
"corrected_text": "stand\u2011still;",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 5,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "punctuation",
"original_text": "view by",
"corrected_text": "view, by",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 6,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "liegelord",
"corrected_text": "liege\u00a0lord",
"observed_line_number": 7,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 7,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "reluctant return",
"corrected_text": "to reluctant return",
"observed_line_number": 3,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": null,
"expected_edit_num": 1,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Chapter XXIX",
"corrected_text": "Chapter XXIX.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 2,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "DREAMING",
"corrected_text": "DREAMING.",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "them,",
"corrected_text": "them;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence",
"corrected_text": "existence,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "it,",
"corrected_text": "it;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "motionless",
"corrected_text": "motionless,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "stand-still,",
"corrected_text": "stand-still;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 8,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "view",
"corrected_text": "view,",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 9,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "existence,",
"corrected_text": "existence;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 10,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "liege-lord",
"corrected_text": "liege lord",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 11,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "of",
"corrected_text": "to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "012"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:37:14.275666",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:30:46.623786",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": 0,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "replacement",
"original_text": "dryest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 6,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction label the change as a \"replacement\", so the edit type matches.\n\n2) Text Content Accuracy: The ground truth changes \"dryest\" \u2192 \"driest\" and inserts \"of\" to yield \"driest of snorts\" (within the larger phrase \"of the ... snorts.\"). The predicted edit also replaces \"dryest snorts\" with \"driest of snorts\", capturing the same core intent (spelling correction plus insertion of \"of\"). The prediction uses a narrower text span and omits surrounding context (\"of the\" and the period), but it does not alter the essential correction. Therefore, the predicted edit matches the ground truth\u2019s intended change."
},
{
"observed_edit_num": 1,
"expected_edit_num": 1,
"tp": 1.0,
"fp": 0.0,
"fn": 0.0,
"type": "italicize",
"original_text": "him",
"corrected_text": "him",
"observed_line_number": 30,
"line_diff": 0,
"line_number_penalty": 0.0,
"judgement": "Edit Type Accuracy: The ground truth edit type is \"italicize\" and the predicted edit type is also \"italicize\", so the type matches exactly.\n\nText Content Accuracy: Both ground truth and prediction apply the italicization to the same word, \"him\" (original_text and corrected_text are identical in both). This captures the core intended change (formatting the word in italics) with no discrepancy."
},
{
"observed_edit_num": 2,
"expected_edit_num": 2,
"tp": 0.9,
"fp": 0.04999999999999999,
"fn": 0.04999999999999999,
"type": "replacement",
"original_text": "authorize",
"corrected_text": "authorise",
"observed_line_number": 38,
"line_diff": 1,
"line_number_penalty": 0.1,
"judgement": "1) Edit Type Accuracy: Both ground truth and prediction specify a \"replacement\" edit type, so the type matches exactly.\n2) Text Content Accuracy: Both edits replace the exact same original text \"authorize\" with the same corrected text \"authorise\", capturing the intended spelling change. Line number differences are to be ignored, so this does not affect correctness."
},
{
"observed_edit_num": null,
"expected_edit_num": 3,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "Clennam:",
"corrected_text": "Clennam;",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 4,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "that-boiling",
"corrected_text": "that\u2014boiling",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 5,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "twenty;",
"corrected_text": "twenty:",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 6,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "punctuation",
"original_text": "now I",
"corrected_text": "now, I",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
},
{
"observed_edit_num": null,
"expected_edit_num": 7,
"tp": 0.0,
"fp": 0.0,
"fn": 1.0,
"type": "replacement",
"original_text": "you now to",
"corrected_text": "you, now, to",
"observed_line_number": null,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False negative: ground truth edit not found in prediction"
}
],
"file_id": "013"
},
{
"model_name": "gpt_o4_mini",
"date": "2026-02-22T15:37:09.083365",
"annotator": "GPT-o4 Mini",
"annotation_date": "2025-04-16T21:59:56.942048",
"details": [
{
"observed_edit_num": 0,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "insertion",
"original_text": "driest snorts",
"corrected_text": "driest of snorts",
"observed_line_number": 5,
"line_diff": null,
"line_number_penalty": 0.0,
"judgement": "False positive: no matching ground truth edit found"
},
{
"observed_edit_num": 1,
"expected_edit_num": null,
"tp": 0.0,
"fp": 1.0,
"fn": 0.0,
"type": "italicize",
"original_text": "benevolence bubbling out of him than out of a ninepin",
"corrected_text": "benevolence bubbling out of him than out of a ninepin